Merge pull request #651 from oist/develop-main

Develop main

Author: milesAraya

.vscode/settings.example.json

@@ -11,6 +11,8 @@
     }
   },
   "eslint.workingDirectories": ["frontend"],
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "editor.formatOnSave": true,
   "flake8.args": ["--config=.flake8"],
   // NOTE: Uncomment following line and fix "optinist_dev" to your conda env name
   // "flake8.path": ["conda", "run", "-n", "optinist_dev", "python", "-m", "flake8"],

Makefile

@@ -1,32 +1,69 @@
+#
+# optinist Makefile
+#
+
+############################## For Testing ##############################
+
+define rm_unused_docker_containers
+	docker ps -a --filter "status=exited" --filter "name=$(1)" --format "{{.ID}}" | xargs --no-run-if-empty docker rm
+endef
+
+PYTEST = poetry run pytest -s
+
 .PHONY: test_run
 test_run:
-	docker-compose -f docker-compose.test.yml down --rmi all --volumes --remove-orphans
-	docker-compose -f docker-compose.test.yml rm -f
-	docker-compose -f docker-compose.test.yml build test_studio
-	docker-compose -f docker-compose.test.yml build test_studio_frontend
-	docker-compose -f docker-compose.test.yml run test_studio
-	docker-compose -f docker-compose.test.yml run test_studio_frontend
-
-.PHONY: test_python
-test_python:
-	docker-compose -f docker-compose.test.yml down --rmi all --volumes --remove-orphans
-	docker-compose -f docker-compose.test.yml rm -f
-	docker-compose -f docker-compose.test.yml build test_studio
-	docker-compose -f docker-compose.test.yml run test_studio
+	# cleanup
+	docker compose -f docker-compose.test.yml down
+	docker compose -f docker-compose.test.yml rm -f
+	@$(call rm_unused_docker_containers, test_studio_backend)
+	# build/run
+	docker compose -f docker-compose.test.yml build test_studio_backend
+	docker compose -f docker-compose.test.yml build test_studio_frontend
+	docker compose -f docker-compose.test.yml run test_studio_backend $(PYTEST) -m "not heavier_processing"
+	docker compose -f docker-compose.test.yml run test_studio_frontend
+
+.PHONY: test_backend
+test_backend:
+	# cleanup
+	docker compose -f docker-compose.test.yml down
+	docker compose -f docker-compose.test.yml rm -f
+	@$(call rm_unused_docker_containers, test_studio_backend)
+	# build/run
+	docker compose -f docker-compose.test.yml build test_studio_backend
+	docker compose -f docker-compose.test.yml run test_studio_backend $(PYTEST) -m "not heavier_processing"
+
+.PHONY: test_backend_full
+test_backend_full:
+	# cleanup
+	docker compose -f docker-compose.test.yml down
+	docker compose -f docker-compose.test.yml rm -f
+	@$(call rm_unused_docker_containers, test_studio_backend)
+	# build/run
+	docker compose -f docker-compose.test.yml build test_studio_backend
+	docker compose -f docker-compose.test.yml run test_studio_backend $(PYTEST)
 
 .PHONY: test_frontend
 test_frontend:
-	docker-compose -f docker-compose.test.yml down --rmi all --volumes --remove-orphans
-	docker-compose -f docker-compose.test.yml rm -f
-	docker-compose -f docker-compose.test.yml build test_studio_frontend
-	docker-compose -f docker-compose.test.yml run test_studio_frontend
+	# cleanup
+	docker compose -f docker-compose.test.yml down
+	docker compose -f docker-compose.test.yml rm -f
+	@$(call rm_unused_docker_containers, test_studio_frontend)
+	# build/run
+	docker compose -f docker-compose.test.yml build test_studio_frontend
+	docker compose -f docker-compose.test.yml run test_studio_frontend
+
+
+############################## For Building ##############################
 
 .PHONY: build_frontend
 build_frontend:
-	docker-compose -f docker-compose.test.yml down --rmi all --volumes --remove-orphans
-	docker-compose -f docker-compose.test.yml rm -f
-	docker-compose -f docker-compose.test.yml build build_studio_frontend
-	docker-compose -f docker-compose.test.yml run build_studio_frontend
+	# cleanup
+	docker compose -f docker-compose.build.yml down
+	docker compose -f docker-compose.build.yml rm -f
+	@$(call rm_unused_docker_containers, studio-build-fe)
+	# build/run
+	docker compose -f docker-compose.build.yml build studio-build-fe
+	docker compose -f docker-compose.build.yml run studio-build-fe
 
 .PHONY: format
 format:
@@ -47,6 +84,9 @@ local_build:
 	cd frontend && yarn install --ignore-scripts && yarn build
 	poetry build
 
+
+############################## For Deployment ##############################
+
 .PHONY: push_testpypi
 push_testpypi:
 	poetry publish -r testpypi

README.md

@@ -34,90 +34,104 @@ OptiNiSt helps researchers try multiple data analysis methods, visualize the res
 OptiNiSt also supports reproducibility of scientific research, standardization of analysis protocols, and developments of novel analysis tools as plug-in.
 
 ## Support library
+
 ### ROI detection
+
 - [x] [Suite2p](https://github.com/MouseLand/suite2p)
 - [x] [CaImAn](https://github.com/flatironinstitute/CaImAn)
 - [x] [LCCD](https://github.com/magnetizedCell/lccd-python)
 
 ### Postprocessing
+
 - [x] Basic Neural Analysis(Event Trigger Average...)
 - [x] Dimension Reduction(PCA...)
 - [x] Neural Decoding(LDA...)
 - [x] Neural Population Analysis(Correlation...)
 
 ### Saving Format
-- [x] [NWB](https://github.com/NeurodataWithoutBorders/pynwb)
 
+- [x] [NWB](https://github.com/NeurodataWithoutBorders/pynwb)
 
 ## Key Features
+
 ### :beginner: Easy-To-Create Workflow
+
 - **zero-knowledge of coding**: OptiNiSt allows you to create analysis pipelines easily on the GUI.
 
 ### :zap: Visualizing analysis results
+
 - **quick visualization**: OptiNiSt supports you visualize the analysis results by plotly.
 
 ### :rocket: Managing Workflows
-- **recording and reproducing**: OptiNiSt records and reproduces the workflow pipelines easily.
 
+- **recording and reproducing**: OptiNiSt records and reproduces the workflow pipelines easily.
 
 ## Installation
-Need anaconda or miniconda environment.
+
+Need anaconda or miniconda or miniforge environment.
+
 ```
 conda create -n optinist python=3.8
 conda activate optinist
 ```
 
 Install from pip.
+
 ```
 pip install optinist
 ```
 
 launch.
+
 ```
 run_optinist
 ```
 
 Open browser. http://localhost:8000
 
 ## Documentation
-https://optinist.readthedocs.io/en/latest/
 
+https://optinist.readthedocs.io/en/latest/
 
 ## Using GUI
+
 ### Workflow
+
 - OptiNiSt allows you to make your analysis pipelines by graph style using nodes and edges on GUI. Parameters for each analysis are easily changeable.
 <p align="center">
   <img width="400px" src="docs/_static/workflow/whole.png" alt="workflow" />
 </p>
 
-
-
 ### Visualize
+
 - OptiNiSt allows you to visualize the analysis results with one click by plotly. It supports a variety of plotting styles.
 <p align="center">
   <img width="400px" src="docs/_static/visualize/whole.png" alt="visualize" />
 </p>
 
 ### Record
+
 - OptiNiSt supports you in recording and reproducing workflow pipelines in an organized manner.
 <p align="center">
   <img width="400px" src="docs/_static/record/whole.png" alt="record" />
 </p>
 
-
-
 ## Contributors
+
 ### Proposers
+
 Kenji Doya, Yukako Yamane [OIST Neural Computation Unit](https://groups.oist.jp/ncu)
 
 ### Main Developers
+
 [Shogo Akiyama](https://github.com/ShogoAkiyama), [Yoshifumi Takeshima](https://github.com/Yoshifumi14)
 
 ### Support Developers
-[Tatsuya Tanabe](https://github.com/ttya16), [Yosuke Kaneko](https://github.com/toto-maru), [Syuya Saeki](https://github.com/hiiaka)
 
+[Tatsuya Tanabe](https://github.com/ttya16), [Yosuke Kaneko](https://github.com/toto-maru), [Syuya Saeki](https://github.com/hiiaka)
 
 ## References
+
 [[Suite2p]](https://github.com/MouseLand/suite2p) Marius Pachitariu, Carsen Stringer, Mario Dipoppa, Sylvia Schröder, L. Federico Rossi, Henry Dalgleish, Matteo Carandini, Kenneth D. Harris. "Suite2p: beyond 10,000 neurons with standard two-photon microscopy". 2017
 [[CaImAn]](https://github.com/flatironinstitute/CaImAn) Andrea Giovannucci Is a corresponding author, Johannes Friedrich, Pat Gunn, Jérémie Kalfon, Brandon L Brown, Sue Ann Koay, Jiannis Taxidis, Farzaneh Najafi, Jeffrey L Gauthier, Pengcheng Zhou, Baljit S Khakh, David W Tank, Dmitri B Chklovskii, Eftychios A Pnevmatikakis. "CaImAn: An open source tool for scalable Calcium Imaging data Analysis". 2019
 [[LCCD]](https://github.com/magnetizedCell/lccd-python) Tsubasa Ito, Keisuke Ota, Kanako Ueno, Yasuhiro Oisi, Chie Matsubara, Kenta Kobayashi, Masamichi Ohkura, Junichi Nakai, Masanori Murayama, Toru Aonishi, "Low computational-cost cell detection method for calcium imaging data", 2022

docker-compose.dev.multiuser.yml

@@ -0,0 +1,58 @@
+version: "3"
+
+services:
+  db:
+    image: mysql:8.4
+    ports:
+      - "127.0.0.1:13306:3306"
+    env_file:
+      - studio/config/.env
+    volumes:
+      - db_data:/var/lib/mysql
+    environment:
+      TZ: Asia/Tokyo
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1"]
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  studio-dev-be:
+    build:
+      context: .
+      dockerfile: studio/config/docker/Dockerfile.dev
+    working_dir: /app
+    volumes:
+      - .:/app
+      # optinist data outputs directories
+      - ../optinist-docker-volumes/.snakemake/:/app/.snakemake
+      - ../optinist-docker-volumes/logs/:/app/logs
+      - ../optinist-docker-volumes/studio_data/:/app/studio_data
+    ports:
+      - "127.0.0.1:8000:8000"
+    command: >
+        bash -c "
+          alembic upgrade head &&
+          poetry run python main.py --reload --host 0.0.0.0 --port 8000
+        "
+    environment:
+      PYTHONPATH: /app/
+      TZ: Asia/Tokyo
+      OPTINIST_DIR: /app/studio_data
+    depends_on:
+      db:
+        condition: service_healthy
+
+  studio-dev-fe:
+    image: node:20.8.0-alpine3.18
+    working_dir: /app/frontend
+    volumes:
+      - ./frontend/:/app/frontend/:cached
+    ports:
+      - "127.0.0.1:3000:3000"
+    command: ash -c 'yarn install && yarn start'
+    environment:
+      TZ: Asia/Tokyo
+
+volumes:
+  db_data:

docker-compose.dev.yml

@@ -8,6 +8,9 @@ services:
     working_dir: /app
     volumes:
       - .:/app
+      # optinist data outputs directories
+      - ../optinist-docker-volumes/.snakemake/:/app/.snakemake
+      - ../optinist-docker-volumes/logs/:/app/logs
       - ../optinist-docker-volumes/studio_data/:/app/studio_data
     ports:
       - "127.0.0.1:8000:8000"

docker-compose.test.yml

@@ -1,16 +1,13 @@
-version: "3.8"
-
 services:
-  test_studio:
-    image: test_studio
-    container_name: test_studio
+  test_studio_backend:
+    image: test_studio_backend
+    container_name: test_studio_backend
     build:
       context: .
       dockerfile: studio/config/docker/Dockerfile.test
     environment:
       PYTHONPATH: .
       TZ: Asia/Tokyo
-    command: bash -c "poetry install --no-root --with test && poetry run python3 -m pytest -s"
     volumes:
       - .:/app
 

docs/README.md

@@ -0,0 +1,12 @@
+## Documentation
+
+### Sphinx
+Make HTML using Sphinx. Used to see changes made when updating readthedocs.
+- [Sphinx documentation](https://docs.readthedocs.io/en/stable/config-file/v2.html)
+
+1. `cd docs`
+2. `pip install -r requirements.txt`
+3. `make html`
+4. Open the generated HTML:
+   - Mac: `open _build/html/index.html`
+   - Windows: `start _build/html/index.html`