initial upload

This commit is contained in:
tom.hempel
2025-09-17 13:19:48 +02:00
commit 68ddcae08b
1436 changed files with 339383 additions and 0 deletions

View File

@ -0,0 +1,16 @@
# README
The documentation for most files can be found inside the file itself.
The most important files are:
- control.py: For setting up the conditions
- app.py: For running the HTML user interface
- server.py: For running the relay server
To setup the experiment environment, you should:
1. Connect the HMDs to same network as the server
2. Start the relay server (i.e. server.py)
3. Start the HTML user interface (e.g. using `fastapi dev app.py`)
4. Communicate the server IP to the HMDs using `TARGET_IP=<HMD IPs> python3 control.py 'IP:<server ip>'`
5. Setup the desired condition (see control.py for more information)
6. Shuffle the word-list (i.e. word-list.txt); on linux this can be achieved using shuf e.g. `shuf word-list.txt > word-list-shuffled.txt`

106
experiment-scripts/app.py Normal file
View File

@ -0,0 +1,106 @@
# Run in dev mode using:
# fastapi dev app.py
#
# After starting the server, you can navigate to http://localhost:8000 to see the web interface.
#
# Note: This requires the user to have the fastapi CLI tool installed.
# The user should be in the same directory as `app.py` as well as `index.html`.
import asyncio
from contextlib import asynccontextmanager
import socket
from fastapi import FastAPI, Request
from fastapi.responses import FileResponse, StreamingResponse
from pydantic import BaseModel
clients = set()
# Broadcast function to notify all SSE clients
async def notify_clients(message: str):
for queue in clients:
await queue.put(message)
async def sock_recvfrom(nonblocking_sock, *pos, loop, **kw):
while True:
try:
return nonblocking_sock.recvfrom(*pos, **kw)
except BlockingIOError:
future = asyncio.Future(loop=loop)
loop.add_reader(nonblocking_sock.fileno(), future.set_result, None)
try:
await future
finally:
loop.remove_reader(nonblocking_sock.fileno())
# Background task: UDP listener
async def udp_listener():
loop = asyncio.get_running_loop()
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(("0.0.0.0", 5001))
sock.setblocking(False)
while True:
data, addr = await sock_recvfrom(sock, 1024, loop=loop)
message = data.decode()
await notify_clients(message)
@asynccontextmanager
async def lifespan(app: FastAPI):
asyncio.create_task(udp_listener())
yield
app = FastAPI(lifespan=lifespan)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# sock.bind(("0.0.0.0", 5002))
SERVER_IP = "127.0.0.1"
SERVER_PORT = 5000
@app.get("/")
async def read_index():
return FileResponse('index.html')
@app.post("/facialexpressions")
def read_item(weights: list[float]):
msg = ';'.join(str(w) for w in weights)
print(len(weights), msg)
sock.sendto(msg.encode('utf-8'), (SERVER_IP, SERVER_PORT))
return { "status": "ok" }
class Word(BaseModel):
target: str
lastWordStatus: int
timeSeconds: float
word: str
@app.post("/word")
def read_word(word: Word):
msg = f"CHARADE:{word.lastWordStatus};{word.timeSeconds};{word.word}"
print(msg)
sock.sendto(msg.encode('utf-8'), (word.target, 5000))
return { "status": "ok" }
# SSE endpoint
@app.get("/news")
async def sse_endpoint(request: Request):
queue = asyncio.Queue()
clients.add(queue)
async def event_generator():
try:
while True:
if await request.is_disconnected():
break
message = await queue.get()
yield f"event: update\ndata: {message}\n\n"
finally:
clients.remove(queue)
return StreamingResponse(event_generator(), media_type="text/event-stream")

View File

@ -0,0 +1,58 @@
import socket
import sys
import os
# Available commands:
#
# Command 'IP'
# ============
# Notify the clients (inside of TARGET_IP) what server they are supposed to
# connect to for sending and retreiving the state of the player, i.e. the
# position of the bones and the face blend shapes.
#
# Example usage:
# TARGET_IP=<list of client ips> python3 control.py 'IP:10.42.0.1'
#
#
# Command 'MODE'
# ==============
# Set the mode of the clients.
# The mode controls which body parts (head and face) are shown as well as
# the interaction method (static or dynamic).
# The mode itself is split into four boolean attributes, which we set
# explicitly to change the mode.
# The boolean values are converted to integer and concatinated using
# semicolon (;).
# The boolean attributes are as follows:
# <show head?>;<show facial exrepssion?>;<show eye rotation?>;<show hands?>
#
# The different conditions are encoded as follows:
#
# | Condition | Mode Command | Additional Notes |
# |--------------------|----------------|--------------------------------------------------------------|
# | Dynamic Face | 'MODE:1;1;1;0' | |
# | Dynamic Hands | 'MODE:0;0;0;1' | |
# | Dynamic Hands+Face | 'MODE:1;1;1;1' | |
# | Static Face | 'MODE:1;0;0;0' | |
# | Static Hands | 'MODE:0;0;0;1' | Same as Dynamic Hands, but the users have to use controllers |
# | Static Hands+Face | 'MODE:1;0;0;1' | For achieving Static Hands the users have to use controllers |
#
# Example usage (set the mode to *Dynamic Hands+Face*):
# TARGET_IP=<list of client ips> python3 control.py 'MODE:1;1;1;1'
val = sys.argv[1]
TARGET_IP=os.getenv('TARGET_IP')
if TARGET_IP is None:
exit(f"You have to set the TARGET_IP environment variable for this to work. Example: TARGET_IP=10.42.0.38,10.42.0.101 python3 {sys.argv[0]} <command>")
for ip in TARGET_IP.split(','):
TARGET_ADDR = (ip, 5000)
if val.startswith("IP:") or val.startswith('MODE:'):
msg = val.encode('utf-8')
print(msg)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.sendto(msg, TARGET_ADDR)
sock.close()
else:
exit("Invalid command")

View File

@ -0,0 +1,542 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Document</title>
<style>
table {
/* width: 100%; */
border-collapse: collapse;
font-family: sans-serif;
font-size: 0.95rem;
background-color: #fff;
border: 1px solid #ccc;
max-height: 400px;
overflow-y: scroll;
margin: 10px;
}
th, td {
padding: 0.25em 1em;
border: 1px solid #ddd;
text-align: left;
}
th {
background-color: #f4f4f4;
font-weight: bold;
}
tr:nth-child(even) {
background-color: #f9f9f9;
}
tr:hover {
background-color: #f1f1f1;
}
div.container table {
max-height: 100vh;
overflow-y: scroll;
}
body {
font-family: Arial, sans-serif;
margin: 20px;
padding: 0;
background-color: #f4f4f4;
}
.container {
max-width: 700px;
min-width: 400px;
margin: 10px;
padding: 20px;
background-color: #fff;
border-radius: 8px;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
}
h2 {
text-align: center;
margin-bottom: 20px;
}
label {
display: block;
margin: 8px 0 4px;
font-weight: bold;
}
select, input[type="number"], input[type="text"] {
width: 100%;
padding: 8px;
margin-bottom: 16px;
border-radius: 4px;
border: 1px solid #ccc;
}
.row {
display: flex;
justify-content: space-between;
}
.row .input-group {
width: 32%;
}
.row .input-group input {
width: 100%;
}
.button-container {
text-align: center;
}
.button-container button {
padding: 10px 20px;
background-color: #4CAF50;
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
}
.button-container button:hover {
background-color: #45a049;
}
.word {
border-radius: 4px;
padding: 3px 8px;
display: inline;
border: 1px solid gray;
}
.word > input[type="checkbox"] {
display: none;
}
.word:hover > input[type="checkbox"] {
display: inline;
}
.word-selected {
background-color:#4c6faf;
}
.word:hover {
border: 1px solid black;
background-color:#0c59e7;
color: white;
font-weight: bold;
}
.word-timer::after {
content: "s";
}
.word-done::before {
content: "❌ ";
}
.word-correct::before {
content: "✅ ";
}
.word:hover .word-correct::before {
content: "";
}
.word:hover .word-done::before {
content: "";
}
.current-word {
font-weight: bold;
}
.current-word::before {
content: "▶ ";
}
#word-list-interactive > div {
margin: 20px 10px;
}
.flex {
display: flex;
justify-content: center;
}
</style>
</head>
<body>
<h1 style="text-align: center;">VR Charades</h1>
<div class="flex">
<!-- <div class="container">
<h2>Facial Expressions</h2>
<div style="max-height: 80vh; overflow-y: scroll;">
<table id="facialExpressions">
<tr>
<th>Index</th>
<th>Property</th>
<th>Value Slider</th>
<th>Value Number</th>
</tr>
</table>
</div>
</div>
<div class="container">
<h2>Modify Element</h2>
<label for="element-select">Select Element</label>
<select id="element-select">
</select>
<div class="row">
<div class="input-group">
<label for="translate-x">Translation X</label>
<input type="number" id="translate-x" placeholder="X">
</div>
<div class="input-group">
<label for="translate-y">Translation Y</label>
<input type="number" id="translate-y" placeholder="Y">
</div>
<div class="input-group">
<label for="translate-z">Translation Z</label>
<input type="number" id="translate-z" placeholder="Z">
</div>
</div>
<div class="row">
<div class="input-group">
<label for="rotate-x">Rotation X</label>
<input type="number" id="rotate-x" placeholder="X">
</div>
<div class="input-group">
<label for="rotate-y">Rotation Y</label>
<input type="number" id="rotate-y" placeholder="Y">
</div>
<div class="input-group">
<label for="rotate-z">Rotation Z</label>
<input type="number" id="rotate-z" placeholder="Z">
</div>
</div>
<div class="button-container">
<button type="button">Modify</button>
</div>
</div> -->
<div class="container">
<h2>Charades</h2>
<label for="ip-player1">IP Player 1</label>
<input type="text" id="ip-player1" placeholder="10.42.0.38">
<label for="ip-player2">IP Player 2</label>
<input type="text" id="ip-player2" placeholder="10.42.0.100">
<fieldset style="display: block; margin: 20px 0px;">
<legend>Current Player For Acting</legend>
<div style="margin: 0.4em;">
<input id="chosen-player-1" name="chosen-player" type="radio" value="player1" checked />
<label style="display: inline; font-weight: normal;" for="chosen-player-1">Player 1</label>
</div>
<div style="margin: 0.4em;">
<input id="chosen-player-2" name="chosen-player" type="radio" value="player2" />
<label style="display: inline; font-weight: normal;" for="chosen-player-2">Player 2</label>
</div>
</fieldset>
<div class="row" style="margin-bottom: 60px;">
<div class="input-group">
<label for="time-s">Time (s)</label>
<input type="number" id="time-s" placeholder="30">
</div>
</div>
<label for="last-word-status">Last Word Status</label>
<select id="last-word-status">
<option value="-1">None</option>
<option value="0">False</option>
<option value="1">Correct</option>
</select>
<div class="row">
<div class="input-group">
<label for="word">Word</label>
<input type="text" id="word" placeholder="Word...">
</div>
</div>
<div class="button-container">
<button type="button" id="button-word">Send</button>
</div>
</div>
<div class="container">
<textarea id="word-list" name="word-list" rows="40" cols="50" placeholder="charade words"></textarea>
<br>
<div class="button-container">
<button type="button" id="button-create-word-items">Modify</button>
</div>
</div>
<div class="container" style="overflow-x: scroll; max-height: 80vh;">
<div id="word-list-interactive">
<p>
Press <b>Modify</b> to generate and run the word list.
</p>
</div>
<div class="button-container">
<button type="button" id="button-save-to-file">Save as CSV</button>
<button type="button" id="button-stop">Stop</button>
</div>
</div>
</div>
<script>
let runningWordIndex = -1;
let runningWordList = [];
function createWordItems() {
const wordList = document.getElementById("word-list");
const text = wordList.value;
const wordListInteractive = document.getElementById("word-list-interactive");
wordListInteractive.innerHTML = "";
runningWordList = [];
runningWordIndex = -1;
let index = 0;
text.trim().split('\n').forEach((word) => {
index += 1;
const div = document.createElement("div");
div.classList.add("word");
div.style.display = "flex";
div.setAttribute("word", word.trim());
const div2 = document.createElement("div");
div2.style.width = "100%";
div2.style.display = "flex";
div2.style.justifyContent = "space-between";
const p = document.createElement("p");
const span = document.createElement("span");
span.innerText = `Word ${index}: `;
span.style.color = "gray";
const wordText = document.createTextNode(word.trim());
p.appendChild(span);
p.appendChild(wordText);
const input = document.createElement("input");
input.setAttribute("type", "checkbox");
input.addEventListener("input", () => {
if (input.checked) {
p.classList.add("word-correct");
} else {
p.classList.remove("word-correct");
}
});
const timerP = document.createElement("p");
timerP.classList.add("word-timer");
const timeSeconds = parseFloat(document.getElementById("time-s").value);
timerP.innerText = timeSeconds.toFixed(2);
timerP.setAttribute("remainingTime", timeSeconds.toFixed(2))
div2.appendChild(p);
div2.appendChild(timerP);
div.appendChild(input);
div.appendChild(div2);
wordListInteractive.appendChild(div);
runningWordList.push(div);
});
}
let frameId = undefined;
let last = undefined;
let newWord;
let lastWordStatus = -1;
function step(timestamp) {
if (last === undefined) {
last = timestamp;
runningWordIndex = 0;
newWord = true;
lastWordStatus = 1;
}
const elapsed = timestamp - last;
last = timestamp;
const runningWord = runningWordList[runningWordIndex];
const p = runningWord.querySelector("p");
const timer = runningWord.querySelector(".word-timer");
let remainingTime = parseFloat(timer.getAttribute("remainingTime")) - (elapsed / 1000);
if (newWord) {
newWord = false;
let word = runningWord.getAttribute("word");
sendNewWord(word, lastWordStatus, remainingTime);
}
if (p.classList.contains("word-correct") || remainingTime < 0) {
if (remainingTime < 0) {
remainingTime = 0;
}
p.classList.add("word-done");
runningWordIndex += 1;
lastWordStatus = p.classList.contains("word-correct") ? 1 : 0;
newWord = true;
}
timer.innerText = remainingTime.toFixed(2);
timer.setAttribute("remainingTime", remainingTime);
if (runningWordIndex < runningWordList.length) {
frameId = requestAnimationFrame(step);
} else {
frameId = undefined;
}
}
document.getElementById("button-save-to-file").addEventListener("click", () => {
let data = "word;correct;time left\n";
for (let i = 0; i < runningWordList.length; ++i) {
const p = runningWordList[i].querySelector("p");
const timer = runningWordList[i].querySelector(".word-timer");
const word = runningWordList[i].getAttribute("word");
const isCorrect = p.classList.contains("word-correct");
const timeLeft = parseFloat(timer.getAttribute("remainingTime"));
data += `${word};${isCorrect};${timeLeft}\n`;
}
window.open('data:text/csv;charset=utf-8,' + escape(data), '_blank');
});
document.getElementById("button-create-word-items").addEventListener("click", () => {
createWordItems();
if (frameId) {
cancelAnimationFrame(frameId);
}
last = undefined;
newWord = undefined;
frameId = requestAnimationFrame(step);
});
document.getElementById('button-word').addEventListener('click', () => {
const chosenPlayer1 = document.getElementById("chosen-player-1").checked;
const chosenPlayer2 = document.getElementById("chosen-player-2").checked;
const ipPlayer1 = document.getElementById("ip-player1").value;
const ipPlayer2 = document.getElementById("ip-player2").value;
const target = chosenPlayer1 ? ipPlayer1 : ipPlayer2;
const targetOther = chosenPlayer1 ? ipPlayer2 : ipPlayer1;
const lastWordStatus = document.getElementById("last-word-status").value;
const timeSeconds = document.getElementById("time-s").value;
const word = document.getElementById("word").value;
const data = {
target: target,
lastWordStatus: parseInt(lastWordStatus),
timeSeconds: parseFloat(timeSeconds),
word: word,
};
fetch("/word", {
method: "POST",
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(data),
}).then(res => {
// console.log("Request complete! response:", res);
});
const data2 = {
target: targetOther,
lastWordStatus: parseInt(lastWordStatus),
timeSeconds: 0.0, // parseFloat(timeSeconds),
word: "", // word,
};
fetch("/word", {
method: "POST",
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(data),
}).then(res => {
// console.log("Request complete! response:", res);
});
});
function sendNewWord(word, lastWordStatus, timeSeconds) {
const chosenPlayer1 = document.getElementById("chosen-player-1").checked;
const chosenPlayer2 = document.getElementById("chosen-player-2").checked;
const ipPlayer1 = document.getElementById("ip-player1").value;
const ipPlayer2 = document.getElementById("ip-player2").value;
const target = chosenPlayer1 ? ipPlayer1 : ipPlayer2;
const targetOther = chosenPlayer1 ? ipPlayer2 : ipPlayer1;
console.log("target:", target);
console.log("targetOther:", targetOther);
const data = {
target: target,
lastWordStatus: lastWordStatus,
timeSeconds,
word: word,
};
fetch("/word", {
method: "POST",
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(data),
}).then(res => {
// console.log("Request complete! response:", res);
});
const dataOther = {
target: targetOther,
lastWordStatus: lastWordStatus,
timeSeconds: 0.0,
word: "", // word,
};
fetch("/word", {
method: "POST",
headers: {'Content-Type': 'application/json'},
body: JSON.stringify(dataOther),
}).then(res => {
// console.log("Request complete! response:", res);
});
}
</script>
</body>
</html>

View File

@ -0,0 +1,51 @@
import socket
import threading
from datetime import datetime
from time import sleep
CONTROL_ADDR = ("127.0.0.1", 5001)
# TODO: Adjust the following addresses so they match the IP addresses of the
# VR headsets.
# In our case the IP addresses were:
# - for player 1: 10.42.0.38
# - for player 2: 10.42.0.72
#
# The ports are hardcoded to 5001 inside the Unity application, so you
# shouldn't change those.
#
# Note: For this to work the VR headsets must be connected to the same network
# as this server.
DEVICE1_ADDR = ("10.42.0.38", 5001)
DEVICE2_ADDR = ("10.42.0.72", 5001)
sock_from_A = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock_from_A.bind(("0.0.0.0", 5000))
def forward(source_socket):
while True:
try:
data, addr = source_socket.recvfrom(1024 * 16)
target_ip = DEVICE1_ADDR[0] if addr[0] == DEVICE2_ADDR[0] else DEVICE2_ADDR[0]
label = "A→B" if addr == DEVICE1_ADDR else "B→A"
if addr != DEVICE1_ADDR and addr != DEVICE2_ADDR:
label = f"unknown {addr}"
sock_from_A.sendto(data, (target_ip, 5000))
timestamp = datetime.now().strftime("%H:%M:%S")
# Logging
#if next(counter) % 20 == 0:
# if addr[0] != DEVICE2_ADDR[0]:
# print(f"[{timestamp}] {label}: {data.decode()}")
# print('sent to ', (target_ip, 5000))
except Exception as e:
print(f"Fehler {label}: {e}")
print("UDP Relay läuft. Strg+C zum Beenden.")
try:
forward(sock_from_A)
except KeyboardInterrupt:
sock_from_A.close()
print("\nUDP Relay beendet.")

View File

@ -0,0 +1,100 @@
Applauding
Archery
Baking a cake
Blowing up a balloon
Boxing
Brushing hair
Brushing teeth
Buttoning a shirt
Calling on the phone
Carrying groceries
Catching a ball
Chopping vegetables
Cleaning a window
Climbing a ladder
Cooking
Crawling
Crying
Cutting hair
Dancing
Digging
Diving into water
Drawing
Drinking
Driving a car
Eating
Falling asleep
Feeding a baby
Fishing
Flying
Folding laundry
Gardening
Giving a gift
Hammering
Hugging
Ironing clothes
Jumping jacks
Juggling
Kayaking
Kicking a ball
Knitting
Laughing
Licking ice cream
Listening to music
Making a bed
Meditating
Mixing a cocktail
Mopping the floor
Opening a jar
Painting
Patching a tire
Petting a dog
Picking flowers
Piano playing
Playing a guitar
Playing ping pong
Pouring a drink
Praying
Pulling a rope
Pushing a cart
Reading a book
Riding a bicycle
Riding a horse
Riding a roller coaster
Running
Saluting
Screaming
Sewing
Shaking hands
Shaving
Shooting a basketball
Shoveling snow
Singing
Sitting down
Skateboarding
Skiing
Sleeping
Slipping on a banana peel
Smelling a flower
Sneezing
Snowball fight
Spitting
Stealing
Stretching
Surfing
Swimming
Swinging
Taking a selfie
Talking on the phone
Tasting soup
Thinking
Throwing a ball
Tying shoelaces
Typing on a keyboard
Using a computer
Walking
Washing hands
Washing a car
Watering plants
Weight lifting
Whispering a secret