add GPT judgment

.gitignore.save

@@ -0,0 +1,153 @@
+bin
+logs
+wandb
+outputs
+data/
+data_local
+.vscode
+_wandb
+
+**/.DS_Store
+
+fuse.cfg
+
+*.ai
+
+# Generation results
+results/
+
+ray/auth.json
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+my_ckpt/
+eval_results/
+task_configs/*
+!task_configs/template.py
+!task_configs/example_task.py
+!task_configs/loader.py
+!task_configs/config_augmentation/
+!task_configs/config_tools/
+test.*
+test/
+demonstrations/
+onnx_output/

chat.py

@@ -0,0 +1,77 @@
+import base64
+import requests
+import cv2
+
+# OpenAI API Key
+
+
+
+# Read text from .txt file
+def read_text_from_file(file_path):
+  with open(file_path, "r") as file:
+    text = file.read()
+    return text
+
+initial_prompt_path = "/home/dyh/OCIA/initial_prompt.txt"
+
+initial_prompt = read_text_from_file(initial_prompt_path)
+
+pass_me_the_bowl_path = '/home/dyh/OCIA/pass_me_the_fruit.txt'
+
+pass_me_the_bowl = read_text_from_file(pass_me_the_bowl_path)
+
+def capture_and_encode_image(camera_index):
+    # Open USB camera
+    cap = cv2.VideoCapture(camera_index)
+    # Read image from camera
+    ret, frame = cap.read()
+    # Release camera
+    cap.release()
+    if not ret:
+       raise ValueError("Failedto capture image")
+    # Convert image to JPEG format
+    _, image_buffer = cv2.imencode('.jpg', frame)
+    # Encode image buffer to base64
+    base64_image = base64.b64encode(image_buffer).decode('utf-8')
+    return base64_image
+
+# Set USB camera index
+camera_index = 2  
+
+# Capture and encode image from USB camera
+base64_image = capture_and_encode_image(camera_index)
+
+headers = {
+  "Content-Type": "application/json",
+  "Authorization": f"Bearer {api_key}"
+}
+
+payload = {
+  "model": "gpt-4o",
+  "messages": [
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": initial_prompt
+        },
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": f"data:image/jpeg;base64,{base64_image}"
+          }
+        },
+        {
+          "type": "text",
+          "text": pass_me_the_bowl
+        }
+      ]
+    }
+  ],
+  "max_tokens": 720
+}
+
+response = requests.post("https://api.openai.com/v1/chat/completions", headers=headers, json=payload)
+
+print(response.json())

chat_with_gpt.py

@@ -0,0 +1,40 @@
+import os
+import base64
+from openai import OpenAI
+
+# Set up proxy if needed
+os.environ["http_proxy"] = "http://localhost:7890"
+os.environ["https_proxy"] = "http://localhost:7890"
+
+def encode_image(image_path):
+  with open(image_path, "rb") as image_file:
+    return base64.b64encode(image_file.read()).decode('utf-8')
+
+# Path to your image
+image_path = "/home/qiuzhi/bowls.png"
+
+# Getting the base64 string
+base64_image = encode_image(image_path)
+
+client = OpenAI()
+
+response = client.chat.completions.create(
+  model="gpt-4o-mini",
+  messages=[
+    {
+      "role": "user",
+      "content": [
+        {"type": "text", "text": "What’s in this image?"},
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": f"data:image/jpeg;base64,{base64_image}",
+          },
+        },
+      ],
+    }
+  ],
+  max_tokens=300,
+)
+
+print(response.choices[0])

data_process/convert_all.py

@@ -55,17 +55,26 @@ def flatten_list(l):
 def flatten_sublist_in_flat_dict(d: dict) -> dict:
     """Flatten the sublist in the flat dict.
     This will change the original dict.
-    Returns the lenth of each value list.
+    Returns the length of each value list.
     """
     trajs_lenth = {}
+    assert isinstance(d, dict), "The input data should be a dict."
     for key, traj in d.items():
+        if not isinstance(traj, (list, tuple)):
+            print(f"Warning: The value for key '{key}' is not a list or tuple, but {type(traj)}")
+            trajs_lenth[key] = 0
+            continue
+
         traj_lenth = len(traj)
         if traj_lenth > 0:
             point = traj[0]
             if isinstance(point, list):
                 if isinstance(point[0], list):
                     for i, p in enumerate(traj):
-                        traj[i] = sum(p, [])
+                        if isinstance(p, list):
+                            traj[i] = sum(p, [])
+                        else:
+                            print(f"Warning: The sublist for key '{key}' at index {i} is not a list, but {type(p)}")
         trajs_lenth[key] = traj_lenth
     return trajs_lenth
 
@@ -292,6 +301,7 @@ def raw_to_dict(
         flatten_mode(str)       -- the flatten mode for the dict keys
         name_converter(dict)    -- the name converter for the flattened dict keys
         concatenater(dict)      -- the concatenate dict to bind several flattened keys together
+        key_filter             -- the key filter to remove the keys
     Returns:
         ep_dicts(dict)          -- the raw data dictionary, with keys as the episode names
     Note:
@@ -320,12 +330,17 @@ def raw_to_dict(
                     )
                 # flatten the dict
                 mode_to_sep_prefix = {
-                    "hdf5": ("/", "/"),
+                    "hdf5": ("/", ""),
                     "hf": (".", ""),
                 }
                 sep_prefix = mode_to_sep_prefix.get(flatten_mode, None)
                 assert sep_prefix is not None, f"Invalid flatten mode {flatten_mode}."
                 data_flat: Dict[str, list] = flatten_dict(raw_data, "", *sep_prefix)
+
+                # 检查 "/observations/stage" 键是否存在
+                if "/observations/stage" not in data_flat:
+                    print(f"Key '/observations/stage' not found in {ep_dir / state_file}")
+
                 # filter the keys
                 if key_filter is not None:
                     for key in key_filter:
@@ -346,6 +361,7 @@ def raw_to_dict(
                 ), "The length of each value list should be the same."
                 for key, value in data_flat.items():
                     name = name_converter.get(key, key)
+                    # print(f"Original key: {key}, Converted name: {name}")
                     ep_dict[name] = pre_process(value)
 
         if video_file_names is not None: