From 7dc57f2001efaa663111363a1dcb6de895791a9b Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 20 Feb 2026 14:10:47 -0800 Subject: [PATCH 01/19] update: create or get material --- utils/api.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/utils/api.py b/utils/api.py index a42a2944..77c7e847 100644 --- a/utils/api.py +++ b/utils/api.py @@ -3,11 +3,13 @@ import os import time import urllib.request -from typing import List +from typing import List, Optional from mat3ra.api_client.endpoints.bank_workflows import BankWorkflowEndpoints from mat3ra.api_client.endpoints.jobs import JobEndpoints +from mat3ra.api_client.endpoints.materials import MaterialEndpoints from mat3ra.api_client.endpoints.properties import PropertiesEndpoints +from mat3ra.api_client.endpoints.workflows import WorkflowEndpoints from tabulate import tabulate @@ -123,3 +125,26 @@ def get_property_by_subworkflow_and_unit_indicies( def get_cluster_name(name: str = "cluster-001") -> str: clusters = json.loads(os.environ.get("CLUSTERS", "[]") or "[]") return clusters[0] if clusters else name + + +def get_or_create_material(endpoint: MaterialEndpoints, material, owner_id: str) -> dict: + """ + Returns an existing material from the collection if one with the same structural hash + exists under the given owner, otherwise creates a new one. + Uses the client-side hash (mat3ra-made Material.hash) to avoid unnecessary DB writes. + + Args: + endpoint (MaterialEndpoints): Material endpoint from the API client. + material: mat3ra-made Material object (must have a .hash property). + owner_id (str): Account ID under which to search and create. + + Returns: + dict: The material dict (existing or newly created). + """ + existing = endpoint.list({"hash": material.hash, "owner._id": owner_id}) + if existing: + print(f"♻️ Reusing already existing Material: {existing[0]['_id']}") + return existing[0] + created = endpoint.create(material.to_dict(), owner_id=owner_id) + print(f"✅ Material created: {created['_id']}") + return created From 33537eab10eb681bdefb76c0b8c2bd5083dc4f97 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 20 Feb 2026 17:52:57 -0800 Subject: [PATCH 02/19] update: use samrt creation --- .../workflows/band_gap.ipynb | 50 +++++++++++++------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/other/materials_designer/workflows/band_gap.ipynb b/other/materials_designer/workflows/band_gap.ipynb index 7162e617..465cdae9 100644 --- a/other/materials_designer/workflows/band_gap.ipynb +++ b/other/materials_designer/workflows/band_gap.ipynb @@ -39,12 +39,13 @@ "\n", "# Material parameters\n", "FOLDER = \"../uploads\"\n", - "MATERIAL_NAME = \"Silicon\"\n", + "MATERIAL_NAME = \"Graphene\" # Name of the material to load from local file or Standata\n", "\n", "# Workflow parameters\n", "WORKFLOW_SEARCH_TERM = \"band_gap.json\"\n", "MY_WORKFLOW_NAME = \"Band Gap Calculation Workflow\"\n", "ADD_RELAXATION = True # Whether to add relaxation subworkflow before band structure calculation\n", + "SAVE_WF_TO_COLLECTION = False # If True, workflow is saved to collection and referenced by ID; if False, embedded into job\n", "\n", "# Model parameters\n", "MODEL_TYPE = \"dft\"\n", @@ -111,7 +112,10 @@ "API_HOST = \"localhost\"\n", "API_PORT = \"3000\"\n", "API_SECURE = \"false\"\n", - "API_VERSION = \"2018-10-01\"" + "API_VERSION = \"2018-10-01\"\n", + "\n", + "OIDC_ACCESS_TOKEN = 'auRTvZOQwFR4q6ydfZQ5WvxYW6RTvcnlV2qoxwE-m47'\n", + "os.environ[\"OIDC_ACCESS_TOKEN\"] = OIDC_ACCESS_TOKEN" ] }, { @@ -124,7 +128,8 @@ "from utils.auth import authenticate\n", "\n", "# Authenticate and have credentials stored in environment variables\n", - "await authenticate()" + "\n", + "# await authenticate()" ] }, { @@ -145,7 +150,8 @@ "source": [ "from mat3ra.api_client import APIClient\n", "\n", - "client = APIClient.authenticate()" + "client = APIClient.authenticate()\n", + "client" ] }, { @@ -206,7 +212,8 @@ "\n", "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", " Materials.get_by_name_first_match(MATERIAL_NAME))\n", - "\n", + "material.basis.coordinates.values[0][0] += 0.0001 # small modification to make it different from the one in Standata and trigger upload to the platform\n", + "print(material.basis.coordinates.values[0])\n", "visualize(material)" ] }, @@ -225,12 +232,13 @@ "metadata": {}, "outputs": [], "source": [ + "from utils.api import get_or_create_material\n", "from utils.generic import dict_to_namespace\n", "\n", "material.basis.set_labels_from_list([])\n", - "saved_material_response = client.materials.create(material.to_dict(), owner_id=ACCOUNT_ID)\n", + "saved_material_response = get_or_create_material(client.materials, material, ACCOUNT_ID)\n", "saved_material = dict_to_namespace(saved_material_response)\n", - "print(f\"✅ Material created: {saved_material._id}\")" + "print(f\"Material ID: {saved_material._id}\")" ] }, { @@ -472,12 +480,18 @@ "metadata": {}, "outputs": [], "source": [ - "workflow_dict = workflow.to_dict()\n", - "\n", - "saved_workflow_response = client.workflows.create(workflow_dict, owner_id=ACCOUNT_ID)\n", + "from utils.generic import dict_to_namespace\n", "\n", - "saved_workflow = dict_to_namespace(saved_workflow_response)\n", - "print(f\"✅ Workflow created: {saved_workflow._id}\")" + "workflow_id_or_dict = None\n", + "if SAVE_WF_TO_COLLECTION:\n", + " from utils.api import get_or_create_workflow\n", + " saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", + " saved_workflow = dict_to_namespace(saved_workflow_response)\n", + " workflow_id_or_dict = saved_workflow._id\n", + " print(f\"Workflow ID: {saved_workflow._id}\")\n", + "else:\n", + " workflow_id_or_dict = workflow.to_dict()\n", + " print(\"Workflow will be embedded into job (not saved to collection)\")" ] }, { @@ -560,21 +574,24 @@ "metadata": {}, "outputs": [], "source": [ + "from utils.api import create_job\n", "from utils.visualize import display_JSON\n", "\n", "material_from_collection = client.materials.get(saved_material._id)\n", "\n", "print(f\"📦 Material: {material_from_collection['_id']}\")\n", - "print(f\"📦 Workflow: {saved_workflow._id}\")\n", + "print(f\"📦 Workflow: {workflow_id_or_dict if SAVE_WF_TO_COLLECTION else '(embedded)'}\")\n", "print(f\"📦 Project: {project_id}\")\n", "\n", - "job_response = client.jobs.create_by_ids(\n", + "job_response = create_job(\n", + " jobs_endpoint=client.jobs,\n", " materials=[vars(saved_material)],\n", - " workflow_id=saved_workflow._id,\n", + " workflow_id_or_dict=workflow_id_or_dict,\n", " project_id=project_id,\n", - " prefix=JOB_NAME,\n", " owner_id=ACCOUNT_ID,\n", + " prefix=JOB_NAME,\n", " compute=compute.to_dict(),\n", + " save_to_collection=SAVE_WF_TO_COLLECTION,\n", ")\n", "\n", "job_dict = job_response[0]\n", @@ -611,6 +628,7 @@ "source": [ "from utils.api import wait_for_jobs_to_finish\n", "\n", + "\n", "wait_for_jobs_to_finish(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" ] }, From 06d71d03203c7196e23e9399ea68a0b5ce89bf54 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Wed, 25 Feb 2026 14:01:53 -0800 Subject: [PATCH 03/19] tmp: update: save entities api --- .../workflows/band_gap.ipynb | 115 +++++++----------- utils/api.py | 79 ++++++++++++ 2 files changed, 125 insertions(+), 69 deletions(-) diff --git a/other/materials_designer/workflows/band_gap.ipynb b/other/materials_designer/workflows/band_gap.ipynb index 1f165a2a..0edf8e5b 100644 --- a/other/materials_designer/workflows/band_gap.ipynb +++ b/other/materials_designer/workflows/band_gap.ipynb @@ -101,40 +101,17 @@ "id": "5", "metadata": {}, "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ[\"API_HOST\"] = \"localhost\"\n", - "os.environ[\"API_PORT\"] = \"3000\"\n", - "os.environ[\"API_SECURE\"] = \"false\"\n", - "\n", - "\n", - "API_HOST = \"localhost\"\n", - "API_PORT = \"3000\"\n", - "API_SECURE = \"false\"\n", - "API_VERSION = \"2018-10-01\"\n", - "\n", - "OIDC_ACCESS_TOKEN = 'auRTvZOQwFR4q6ydfZQ5WvxYW6RTvcnlV2qoxwE-m47'\n", - "os.environ[\"OIDC_ACCESS_TOKEN\"] = OIDC_ACCESS_TOKEN" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], "source": [ "from utils.auth import authenticate\n", "\n", "# Authenticate and have credentials stored in environment variables\n", "\n", - "# await authenticate()" + "await authenticate()" ] }, { "cell_type": "markdown", - "id": "7", + "id": "6", "metadata": {}, "source": [ "### 2.2. Initialize API client\n", @@ -144,7 +121,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -156,7 +133,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "8", "metadata": {}, "source": [ "### 2.3. Select account to work under\n", @@ -166,7 +143,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -176,7 +153,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -191,7 +168,7 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "11", "metadata": {}, "source": [ "## 3. Create material\n", @@ -201,7 +178,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -212,14 +189,13 @@ "\n", "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", " Materials.get_by_name_first_match(MATERIAL_NAME))\n", - "material.basis.coordinates.values[0][0] += 0.0001 # small modification to make it different from the one in Standata and trigger upload to the platform\n", - "print(material.basis.coordinates.values[0])\n", + "\n", "visualize(material)" ] }, { "cell_type": "markdown", - "id": "14", + "id": "13", "metadata": {}, "source": [ "### 3.2. Save material to the platform" @@ -228,7 +204,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -243,7 +219,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "15", "metadata": { "tags": [] }, @@ -254,7 +230,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -263,7 +239,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "17", "metadata": {}, "source": [ "## 5. Create workflow and set its parameters\n", @@ -273,7 +249,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -286,7 +262,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -297,7 +273,7 @@ }, { "cell_type": "markdown", - "id": "21", + "id": "20", "metadata": {}, "source": [ "### 5.2. Create workflow from standard workflows and preview it" @@ -306,7 +282,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -322,7 +298,7 @@ }, { "cell_type": "markdown", - "id": "23", + "id": "22", "metadata": {}, "source": [ "### 5.3. Add relaxation subworkflow" @@ -331,7 +307,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -345,7 +321,7 @@ }, { "cell_type": "markdown", - "id": "25", + "id": "24", "metadata": {}, "source": [ "### 5.4. Change subworkflow details (Model subtype)\n", @@ -355,7 +331,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -368,7 +344,7 @@ }, { "cell_type": "markdown", - "id": "27", + "id": "26", "metadata": {}, "source": [ "#### 5.4.2. Modify model in subworkflow units" @@ -377,7 +353,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -407,7 +383,7 @@ }, { "cell_type": "markdown", - "id": "29", + "id": "28", "metadata": {}, "source": [ "### 5.5. Modify k-grid in subworkflow units\n", @@ -417,7 +393,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -430,7 +406,7 @@ }, { "cell_type": "markdown", - "id": "31", + "id": "30", "metadata": {}, "source": [ "#### 5.5.3. Modify workflow units with new context" @@ -439,7 +415,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -467,7 +443,7 @@ }, { "cell_type": "markdown", - "id": "33", + "id": "32", "metadata": {}, "source": [ "### 5.6. Save workflow to collection" @@ -476,7 +452,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -496,7 +472,7 @@ }, { "cell_type": "markdown", - "id": "35", + "id": "34", "metadata": {}, "source": [ "## 6. Create the compute configuration\n", @@ -506,16 +482,17 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "35", "metadata": {}, "outputs": [], "source": [ - "clusters = client.clusters.list()" + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c.fqdn for c in clusters]}\")" ] }, { "cell_type": "markdown", - "id": "37", + "id": "36", "metadata": {}, "source": [ "### 6.2. Create compute configuration for the job" @@ -524,7 +501,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -541,7 +518,7 @@ }, { "cell_type": "markdown", - "id": "39", + "id": "38", "metadata": {}, "source": [ "## 7. Create the job with material and workflow configuration\n", @@ -551,7 +528,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -561,7 +538,7 @@ }, { "cell_type": "markdown", - "id": "41", + "id": "40", "metadata": {}, "source": [ "### 7.2. Create job" @@ -570,7 +547,7 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "41", "metadata": {}, "outputs": [], "source": [ @@ -603,7 +580,7 @@ }, { "cell_type": "markdown", - "id": "43", + "id": "42", "metadata": {}, "source": [ "## 8. Submit the job and monitor the status" @@ -612,7 +589,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -622,7 +599,7 @@ { "cell_type": "code", "execution_count": null, - "id": "45", + "id": "44", "metadata": {}, "outputs": [], "source": [ @@ -634,7 +611,7 @@ }, { "cell_type": "markdown", - "id": "46", + "id": "45", "metadata": {}, "source": [ "## 9. Retrieve results" @@ -643,7 +620,7 @@ { "cell_type": "code", "execution_count": null, - "id": "47", + "id": "46", "metadata": {}, "outputs": [], "source": [ @@ -657,7 +634,7 @@ { "cell_type": "code", "execution_count": null, - "id": "48", + "id": "47", "metadata": {}, "outputs": [], "source": [] diff --git a/utils/api.py b/utils/api.py index 77c7e847..d8ada34d 100644 --- a/utils/api.py +++ b/utils/api.py @@ -148,3 +148,82 @@ def get_or_create_material(endpoint: MaterialEndpoints, material, owner_id: str) created = endpoint.create(material.to_dict(), owner_id=owner_id) print(f"✅ Material created: {created['_id']}") return created + + +def get_or_create_workflow(endpoint: WorkflowEndpoints, workflow, owner_id: str) -> dict: + """ + Creates the workflow on the server, then uses the server-assigned hash to check for + pre-existing duplicates. If a duplicate exists, deletes the new entry and returns the + original. The server is the authoritative source for structural deduplication. + + Args: + endpoint (WorkflowEndpoints): Workflow endpoint from the API client. + workflow: mat3ra-wode Workflow object with a .to_dict() method. + owner_id (str): Account ID under which to search and create. + + Returns: + dict: The workflow dict (existing or newly created). + """ + # TODO: calculate the hash in wode, client side + created = endpoint.create(workflow.to_dict(), owner_id=owner_id) + duplicates = endpoint.list({"hash": created["hash"], "owner._id": owner_id}) + if len(duplicates) > 1: + original = next(w for w in duplicates if w["_id"] != created["_id"]) + endpoint.delete(created["_id"]) + print(f"♻️ Workflow already exists: {original['_id']}") + return original + print(f"✅ Workflow created: {created['_id']}") + return created + + +def create_job( + jobs_endpoint: JobEndpoints, + materials: List[dict], + workflow_id_or_dict, + project_id: str, + owner_id: str, + prefix: str, + compute: Optional[dict] = None, + save_to_collection: bool = True, +) -> List[dict]: + """ + Creates jobs for each material using either collection references or an embedded workflow. + + Args: + jobs_endpoint (JobEndpoints): Job endpoint from the API client. + materials (list[dict]): List of material dicts (must include _id and formula). + workflow_id_or_dict: Workflow _id (str) if save_to_collection=True, + or full workflow dict if save_to_collection=False. + project_id (str): Project ID. + owner_id (str): Account ID. + prefix (str): Job name prefix. + compute (dict, optional): Compute configuration dict. + save_to_collection (bool): If True, uses create_by_ids; otherwise embeds the workflow. + + Returns: + list[dict]: List of created job dicts. + """ + if save_to_collection: + return jobs_endpoint.create_by_ids( + materials=materials, + workflow_id=workflow_id_or_dict, + project_id=project_id, + prefix=prefix, + owner_id=owner_id, + compute=compute, + ) + jobs = [] + for material in materials: + job_name = " ".join((prefix, material["formula"])) + embedded_workflow = {k: v for k, v in workflow_id_or_dict.items() if k != "_id"} + config = { + "_project": {"_id": project_id}, + "workflow": embedded_workflow, + "_material": {"_id": material["_id"]}, + "owner": {"_id": owner_id}, + "name": job_name, + } + if compute: + config["compute"] = compute + jobs.append(jobs_endpoint.create(config)) + return jobs From d1abb3c3bc3066eaa94420070b5e70d387444f9b Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 26 Feb 2026 10:51:22 -0800 Subject: [PATCH 04/19] feat: add total energy nb --- .../workflows/total_energy.ipynb | 595 ++++++++++++++++++ 1 file changed, 595 insertions(+) create mode 100644 other/materials_designer/workflows/total_energy.ipynb diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb new file mode 100644 index 00000000..01b7942c --- /dev/null +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -0,0 +1,595 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Total Energy\n", + "\n", + "This notebook demonstrates how to run a workflow calculation using the Mat3ra API.\n", + "\n", + "## Process Overview\n", + "1. Set up environment and parameters\n", + "2. Authenticate and initialize API client\n", + "3. Select account and project\n", + "4. Load and save materials\n", + "5. Configure workflow\n", + "6. Set up compute resources\n", + "7. Create and submit job\n", + "8. Monitor execution\n", + "9. Retrieve and visualize results" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "if sys.platform == \"emscripten\":\n", + " import micropip\n", + "\n", + " await micropip.install(\"mat3ra-api-examples\", deps=False)\n", + " await micropip.install(\"mat3ra-utils\")\n", + " from mat3ra.utils.jupyterlite.packages import install_packages\n", + "\n", + " await install_packages(\"api_examples\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.1. Set parameters and configurations for the workflow and job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "# Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAME = \"Silicon\" # Name of the material to load from local file or Standata\n", + "\n", + "# 4. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", + "MY_WORKFLOW_NAME = \"Total Energy\"\n", + "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", + "# ADD_RELAXATION = True # Whether to add relaxation subworkflow\n", + "SAVE_WF_TO_COLLECTION = False # If True, workflow is saved to collection\n", + "\n", + "# MODEL_TYPE = \"dft\"\n", + "# MODEL_SUBTYPE = \"gga\" # or \"lda\"\n", + "# MODEL_FUNCTIONAL = \"pbe\" # for gga: pbe, for lda: pz\n", + "\n", + "# KGRID = [4, 4, 4] # k-grid for main calculation\n", + "# RELAXATION_KGRID = [2, 2, 2] # k-grid for relaxation (if applicable)\n", + "\n", + "# 5. Compute parameters\n", + "CLUSTER_NAME = None # specify i.e. \"cluster-001\" to use that cluster\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 2\n", + "\n", + "# 6. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "JOB_NAME = f\"Total Energy {timestamp}\"\n", + "POLL_INTERVAL = 30 # seconds\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "### 1.2. API Configuration (optional, for local development)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"API_HOST\"] = \"localhost\"\n", + "os.environ[\"API_PORT\"] = \"3000\"\n", + "os.environ[\"API_SECURE\"] = \"false\"\n", + "\n", + "API_HOST = \"localhost\"\n", + "API_PORT = \"3000\"\n", + "API_SECURE = \"false\"\n", + "API_VERSION = \"2018-10-01\"\n", + "\n", + "OIDC_ACCESS_TOKEN = '_qHzr2miPGLP3xlnV-n2E_k9Wdr_3xVcbcgztVwk-Sx'\n", + "os.environ[\"OIDC_ACCESS_TOKEN\"] = OIDC_ACCESS_TOKEN" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.auth import authenticate\n", + "\n", + "# Authenticate in the browser and have credentials stored in environment variables\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.2. Initialize API Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": { + "jupyter": { + "is_executing": true + } + }, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "### 2.4. Select project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "## 3. Create material\n", + "### 3.1. Load material from local file (or Standata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from utils.visualize import visualize_materials as visualize\n", + "from utils.jupyterlite import load_material_from_folder\n", + "\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", + " Materials.get_by_name_first_match(MATERIAL_NAME))\n", + "\n", + "visualize(material)" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "### 3.2. Save material to the platform" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.api import get_or_create_material\n", + "from utils.generic import dict_to_namespace\n", + "\n", + "saved_material_response = get_or_create_material(client.materials, material, ACCOUNT_ID)\n", + "saved_material = dict_to_namespace(saved_material_response)" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "## 4. Create workflow and set its parameters\n", + "### 4.1. Get list of applications and select one" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.applications import ApplicationStandata\n", + "from mat3ra.ade.application import Application\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "print(f\"Using application: {app.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.2. Create workflow from standard workflows and preview it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "from utils.visualize import visualize_workflow\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "workflow = Workflow.create(workflow_config)\n", + "workflow.name = MY_WORKFLOW_NAME\n", + "\n", + "visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "### 4.3. Modify workflow (Optional)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: Add workflow modifications here\n", + "\n", + "# Example: Add relaxation subworkflow\n", + "# if ADD_RELAXATION:\n", + "# workflow.add_relaxation()\n", + "\n", + "# Example: Change model parameters\n", + "# from mat3ra.mode import Model\n", + "# from mat3ra.standata.model_tree import ModelTreeStandata\n", + "#\n", + "# model_config = ModelTreeStandata.get_model_by_parameters(\n", + "# type=MODEL_TYPE,\n", + "# subtype=MODEL_SUBTYPE,\n", + "# functional={\"slug\": MODEL_FUNCTIONAL},\n", + "# )\n", + "# model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"us\"}\n", + "# model = Model.create(model_config)\n", + "#\n", + "# for subworkflow in workflow.subworkflows:\n", + "# subworkflow.model = model\n", + "\n", + "# Example: Modify k-grids\n", + "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "#\n", + "# new_context = PointsGridDataProvider(dimensions=KGRID, isEdited=True).yield_data()\n", + "# subworkflow = workflow.subworkflows[0]\n", + "# unit = subworkflow.get_unit_by_name(name=\"pw_scf\") # Adjust unit name as needed\n", + "# unit.add_context(new_context)\n", + "# subworkflow.set_unit(unit)\n", + "\n", + "# Preview modified workflow\n", + "# visualize_workflow(workflow)\n" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "### 4.4. Save workflow to collection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.generic import dict_to_namespace\n", + "\n", + "workflow_id_or_dict = None\n", + "\n", + "if SAVE_WF_TO_COLLECTION:\n", + " from utils.api import get_or_create_workflow\n", + "\n", + " saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", + " saved_workflow = dict_to_namespace(saved_workflow_response)\n", + " workflow_id_or_dict = saved_workflow._id\n", + " print(f\"Workflow ID: {saved_workflow._id}\")\n", + "else:\n", + " workflow_id_or_dict = workflow.to_dict()\n", + " print(\"Workflow will be embedded into job (not saved to collection)\")" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "## 5. Create the compute configuration\n", + "### 5.1. Get list of clusters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c.fqdn for c in clusters]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "### 5.2. Create compute configuration for the job\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "# Select first available cluster or use specified name\n", + "cluster = next((c for c in clusters if c.fqdn == CLUSTER_NAME), clusters[0] if clusters else None)\n", + "\n", + "compute = Compute(\n", + " cluster=cluster,\n", + " queue=QUEUE_NAME,\n", + " ppn=PPN\n", + ")\n", + "\n", + "print(f\"Using cluster: {cluster.fqdn}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "## 6. Create the job with material and workflow configuration\n", + "### 6.1. Create job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.api import create_job\n", + "from utils.visualize import display_JSON\n", + "\n", + "print(f\"Material: {saved_material._id}\")\n", + "print(f\"Workflow: {workflow_id_or_dict if SAVE_WF_TO_COLLECTION else '(embedded)'}\")\n", + "print(f\"Project: {project_id}\")\n", + "\n", + "job_response = create_job(\n", + " jobs_endpoint=client.jobs,\n", + " materials=[vars(saved_material)],\n", + " workflow_id_or_dict=workflow_id_or_dict,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=JOB_NAME,\n", + " compute=compute.to_dict(),\n", + " save_to_collection=SAVE_WF_TO_COLLECTION,\n", + ")\n", + "\n", + "job_dict = job_response[0]\n", + "job = dict_to_namespace(job_dict)\n", + "job_id = job._id\n", + "print(\"✅ Job created successfully!\")\n", + "print(f\"Job ID: {job_id}\")\n", + "display_JSON(job_response)" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "## 7. Submit the job and monitor the status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "client.jobs.submit(job_id)\n", + "print(f\"✅ Job {job_id} submitted successfully!\")" + ] + }, + { + "cell_type": "markdown", + "id": "36", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.api import wait_for_jobs_to_finish\n", + "\n", + "wait_for_jobs_to_finish(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" + ] + }, + { + "cell_type": "markdown", + "id": "38", + "metadata": {}, + "source": [ + "## 8. Retrieve results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: Specify property extraction and visualization\n", + "\n", + "# Example: Extract a specific property\n", + "# from mat3ra.prode import PropertyName\n", + "# from utils.visualize import render_prove\n", + "#\n", + "# property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy)\n", + "# render_prove(property_data, title=\"Total Energy\")\n", + "\n", + "# Example: Get all properties\n", + "# properties = client.properties.list({\"_job._id\": job_id})\n", + "# for prop in properties:\n", + "# print(f\"{prop['name']}: {prop.get('data', 'N/A')}\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 17afe31175ab8c79d11ee11936d9329164a6c4dc Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 27 Feb 2026 12:40:43 -0800 Subject: [PATCH 05/19] update: fully working TE NB --- .../workflows/total_energy.ipynb | 51 +++++++------------ 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 01b7942c..357657da 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -72,7 +72,6 @@ "ORGANIZATION_NAME = None\n", "\n", "# 3. Material parameters\n", - "# Material parameters\n", "FOLDER = \"../uploads\"\n", "MATERIAL_NAME = \"Silicon\" # Name of the material to load from local file or Standata\n", "\n", @@ -80,15 +79,8 @@ "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", "MY_WORKFLOW_NAME = \"Total Energy\"\n", "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", - "# ADD_RELAXATION = True # Whether to add relaxation subworkflow\n", - "SAVE_WF_TO_COLLECTION = False # If True, workflow is saved to collection\n", - "\n", - "# MODEL_TYPE = \"dft\"\n", - "# MODEL_SUBTYPE = \"gga\" # or \"lda\"\n", - "# MODEL_FUNCTIONAL = \"pbe\" # for gga: pbe, for lda: pz\n", - "\n", - "# KGRID = [4, 4, 4] # k-grid for main calculation\n", - "# RELAXATION_KGRID = [2, 2, 2] # k-grid for relaxation (if applicable)\n", + "ADD_RELAXATION = True # Whether to add relaxation subworkflow\n", + "SAVE_WF_TO_COLLECTION = True # If True, workflow is saved to collection\n", "\n", "# 5. Compute parameters\n", "CLUSTER_NAME = None # specify i.e. \"cluster-001\" to use that cluster\n", @@ -128,7 +120,7 @@ "API_SECURE = \"false\"\n", "API_VERSION = \"2018-10-01\"\n", "\n", - "OIDC_ACCESS_TOKEN = '_qHzr2miPGLP3xlnV-n2E_k9Wdr_3xVcbcgztVwk-Sx'\n", + "OIDC_ACCESS_TOKEN = 'nAzK2neRGJ-_2zGDzLdEsFNr6E1yjBw8pawWVUCZ3fj'\n", "os.environ[\"OIDC_ACCESS_TOKEN\"] = OIDC_ACCESS_TOKEN" ] }, @@ -166,11 +158,7 @@ "cell_type": "code", "execution_count": null, "id": "10", - "metadata": { - "jupyter": { - "is_executing": true - } - }, + "metadata": {}, "outputs": [], "source": [ "from mat3ra.api_client import APIClient\n", @@ -429,7 +417,7 @@ "outputs": [], "source": [ "clusters = client.clusters.list()\n", - "print(f\"Available clusters: {[c.fqdn for c in clusters]}\")" + "print(f\"Available clusters: {[c for c in clusters]}\")" ] }, { @@ -450,15 +438,14 @@ "from mat3ra.ide.compute import Compute\n", "\n", "# Select first available cluster or use specified name\n", - "cluster = next((c for c in clusters if c.fqdn == CLUSTER_NAME), clusters[0] if clusters else None)\n", + "cluster = next((c for c in clusters if c[\"hostname\"] == CLUSTER_NAME), clusters[0] if clusters else None)\n", "\n", "compute = Compute(\n", " cluster=cluster,\n", " queue=QUEUE_NAME,\n", " ppn=PPN\n", ")\n", - "\n", - "print(f\"Using cluster: {cluster.fqdn}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + "compute.to_dict()" ] }, { @@ -555,20 +542,20 @@ "metadata": {}, "outputs": [], "source": [ - "# TODO: Specify property extraction and visualization\n", - "\n", - "# Example: Extract a specific property\n", - "# from mat3ra.prode import PropertyName\n", - "# from utils.visualize import render_prove\n", - "#\n", - "# property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy)\n", - "# render_prove(property_data, title=\"Total Energy\")\n", + "from mat3ra.prode import PropertyName\n", + "from utils.visualize import visualize_properties\n", "\n", - "# Example: Get all properties\n", - "# properties = client.properties.list({\"_job._id\": job_id})\n", - "# for prop in properties:\n", - "# print(f\"{prop['name']}: {prop.get('data', 'N/A')}\")\n" + "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy)\n", + "visualize_properties(property_data, title=\"Total Energy\")\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 5d7a9d4870351ecebe0dee74b775fd1231450a69 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 27 Feb 2026 12:40:56 -0800 Subject: [PATCH 06/19] update: fully working TE NB --- .../workflows/total_energy.ipynb | 90 +++++++------------ 1 file changed, 34 insertions(+), 56 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 357657da..8e7d3197 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -102,31 +102,9 @@ "### 1.2. API Configuration (optional, for local development)" ] }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ[\"API_HOST\"] = \"localhost\"\n", - "os.environ[\"API_PORT\"] = \"3000\"\n", - "os.environ[\"API_SECURE\"] = \"false\"\n", - "\n", - "API_HOST = \"localhost\"\n", - "API_PORT = \"3000\"\n", - "API_SECURE = \"false\"\n", - "API_VERSION = \"2018-10-01\"\n", - "\n", - "OIDC_ACCESS_TOKEN = 'nAzK2neRGJ-_2zGDzLdEsFNr6E1yjBw8pawWVUCZ3fj'\n", - "os.environ[\"OIDC_ACCESS_TOKEN\"] = OIDC_ACCESS_TOKEN" - ] - }, { "cell_type": "markdown", - "id": "7", + "id": "6", "metadata": {}, "source": [ "## 2. Authenticate and initialize API client\n", @@ -136,7 +114,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -148,7 +126,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "8", "metadata": {}, "source": [ "### 2.2. Initialize API Client" @@ -157,7 +135,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -169,7 +147,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "10", "metadata": {}, "source": [ "### 2.3. Select account to work under" @@ -178,7 +156,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -188,7 +166,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -203,7 +181,7 @@ }, { "cell_type": "markdown", - "id": "14", + "id": "13", "metadata": {}, "source": [ "### 2.4. Select project" @@ -212,7 +190,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -223,7 +201,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "15", "metadata": {}, "source": [ "## 3. Create material\n", @@ -233,7 +211,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -250,7 +228,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "17", "metadata": {}, "source": [ "### 3.2. Save material to the platform" @@ -259,7 +237,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -272,7 +250,7 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "19", "metadata": {}, "source": [ "## 4. Create workflow and set its parameters\n", @@ -282,7 +260,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -296,7 +274,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "21", "metadata": {}, "source": [ "### 4.2. Create workflow from standard workflows and preview it" @@ -305,7 +283,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -322,7 +300,7 @@ }, { "cell_type": "markdown", - "id": "24", + "id": "23", "metadata": {}, "source": [ "### 4.3. Modify workflow (Optional)" @@ -331,7 +309,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -371,7 +349,7 @@ }, { "cell_type": "markdown", - "id": "26", + "id": "25", "metadata": {}, "source": [ "### 4.4. Save workflow to collection" @@ -380,7 +358,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -402,7 +380,7 @@ }, { "cell_type": "markdown", - "id": "28", + "id": "27", "metadata": {}, "source": [ "## 5. Create the compute configuration\n", @@ -412,7 +390,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -422,7 +400,7 @@ }, { "cell_type": "markdown", - "id": "30", + "id": "29", "metadata": {}, "source": [ "### 5.2. Create compute configuration for the job\n" @@ -431,7 +409,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "30", "metadata": {}, "outputs": [], "source": [ @@ -450,7 +428,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "31", "metadata": {}, "source": [ "## 6. Create the job with material and workflow configuration\n", @@ -460,7 +438,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -492,7 +470,7 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "33", "metadata": {}, "source": [ "## 7. Submit the job and monitor the status" @@ -501,7 +479,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -511,14 +489,14 @@ }, { "cell_type": "markdown", - "id": "36", + "id": "35", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -529,7 +507,7 @@ }, { "cell_type": "markdown", - "id": "38", + "id": "37", "metadata": {}, "source": [ "## 8. Retrieve results" @@ -538,7 +516,7 @@ { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -552,7 +530,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "39", "metadata": {}, "outputs": [], "source": [] From f1ec0c70198a16c4e16bd70c785636f03698e282 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Sun, 1 Mar 2026 14:37:40 -0800 Subject: [PATCH 07/19] update: calcualte hash --- utils/api.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/utils/api.py b/utils/api.py index d8ada34d..2ee049bb 100644 --- a/utils/api.py +++ b/utils/api.py @@ -164,14 +164,11 @@ def get_or_create_workflow(endpoint: WorkflowEndpoints, workflow, owner_id: str) Returns: dict: The workflow dict (existing or newly created). """ - # TODO: calculate the hash in wode, client side + existing = endpoint.list({"hash": workflow.hash, "owner._id": owner_id}) + if existing: + print(f"♻️ Reusing already existing Workflow: {existing[0]['_id']}") + return existing[0] created = endpoint.create(workflow.to_dict(), owner_id=owner_id) - duplicates = endpoint.list({"hash": created["hash"], "owner._id": owner_id}) - if len(duplicates) > 1: - original = next(w for w in duplicates if w["_id"] != created["_id"]) - endpoint.delete(created["_id"]) - print(f"♻️ Workflow already exists: {original['_id']}") - return original print(f"✅ Workflow created: {created['_id']}") return created From 190b36e4432dde7ee5d24d3aec2d767ff18939f5 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Sun, 1 Mar 2026 16:25:11 -0800 Subject: [PATCH 08/19] update: cleanup --- other/materials_designer/workflows/total_energy.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 8e7d3197..ef960186 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -141,8 +141,7 @@ "source": [ "from mat3ra.api_client import APIClient\n", "\n", - "client = APIClient.authenticate()\n", - "client" + "client = APIClient.authenticate()" ] }, { @@ -295,6 +294,7 @@ "workflow = Workflow.create(workflow_config)\n", "workflow.name = MY_WORKFLOW_NAME\n", "\n", + "print(workflow.hash)\n", "visualize_workflow(workflow)" ] }, From cc178655a8941c0c58e07eb26a856f623cdabd1d Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 2 Mar 2026 19:15:33 -0800 Subject: [PATCH 09/19] update: cleanup --- .../workflows/total_energy.ipynb | 116 ++++++++---------- 1 file changed, 48 insertions(+), 68 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index ef960186..109da065 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -17,8 +17,7 @@ "5. Configure workflow\n", "6. Set up compute resources\n", "7. Create and submit job\n", - "8. Monitor execution\n", - "9. Retrieve and visualize results" + "8. Retrieve and visualize results" ] }, { @@ -54,7 +53,7 @@ "id": "3", "metadata": {}, "source": [ - "### 1.1. Set parameters and configurations for the workflow and job" + "### 1.2. Set parameters and configurations for the workflow and job" ] }, { @@ -79,33 +78,24 @@ "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", "MY_WORKFLOW_NAME = \"Total Energy\"\n", "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", - "ADD_RELAXATION = True # Whether to add relaxation subworkflow\n", + "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", "SAVE_WF_TO_COLLECTION = True # If True, workflow is saved to collection\n", "\n", "# 5. Compute parameters\n", "CLUSTER_NAME = None # specify i.e. \"cluster-001\" to use that cluster\n", "QUEUE_NAME = QueueName.D\n", - "PPN = 2\n", + "PPN = 1\n", "\n", "# 6. Job parameters\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", "JOB_NAME = f\"Total Energy {timestamp}\"\n", - "POLL_INTERVAL = 30 # seconds\n", - "\n" + "POLL_INTERVAL = 30 # seconds\n" ] }, { "cell_type": "markdown", "id": "5", "metadata": {}, - "source": [ - "### 1.2. API Configuration (optional, for local development)" - ] - }, - { - "cell_type": "markdown", - "id": "6", - "metadata": {}, "source": [ "## 2. Authenticate and initialize API client\n", "### 2.1. Authenticate" @@ -114,7 +104,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "6", "metadata": {}, "outputs": [], "source": [ @@ -126,7 +116,7 @@ }, { "cell_type": "markdown", - "id": "8", + "id": "7", "metadata": {}, "source": [ "### 2.2. Initialize API Client" @@ -135,7 +125,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -146,7 +136,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "### 2.3. Select account to work under" @@ -155,7 +145,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -165,7 +155,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -180,7 +170,7 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "12", "metadata": {}, "source": [ "### 2.4. Select project" @@ -189,7 +179,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -200,7 +190,7 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "14", "metadata": {}, "source": [ "## 3. Create material\n", @@ -210,7 +200,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -227,7 +217,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "16", "metadata": {}, "source": [ "### 3.2. Save material to the platform" @@ -236,7 +226,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -249,7 +239,7 @@ }, { "cell_type": "markdown", - "id": "19", + "id": "18", "metadata": {}, "source": [ "## 4. Create workflow and set its parameters\n", @@ -259,7 +249,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -273,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "21", + "id": "20", "metadata": {}, "source": [ "### 4.2. Create workflow from standard workflows and preview it" @@ -282,7 +272,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -294,13 +284,12 @@ "workflow = Workflow.create(workflow_config)\n", "workflow.name = MY_WORKFLOW_NAME\n", "\n", - "print(workflow.hash)\n", "visualize_workflow(workflow)" ] }, { "cell_type": "markdown", - "id": "23", + "id": "22", "metadata": {}, "source": [ "### 4.3. Modify workflow (Optional)" @@ -309,24 +298,23 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ - "# TODO: Add workflow modifications here\n", - "\n", "# Example: Add relaxation subworkflow\n", - "# if ADD_RELAXATION:\n", - "# workflow.add_relaxation()\n", + "if ADD_RELAXATION:\n", + " workflow.add_relaxation()\n", "\n", - "# Example: Change model parameters\n", + "\n", + "# # Example: Change model parameters\n", "# from mat3ra.mode import Model\n", "# from mat3ra.standata.model_tree import ModelTreeStandata\n", "#\n", "# model_config = ModelTreeStandata.get_model_by_parameters(\n", - "# type=MODEL_TYPE,\n", - "# subtype=MODEL_SUBTYPE,\n", - "# functional={\"slug\": MODEL_FUNCTIONAL},\n", + "# type=\"dft\",\n", + "# subtype=\"gga\",\n", + "# functional=\"pbe\"\n", "# )\n", "# model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"us\"}\n", "# model = Model.create(model_config)\n", @@ -334,7 +322,7 @@ "# for subworkflow in workflow.subworkflows:\n", "# subworkflow.model = model\n", "\n", - "# Example: Modify k-grids\n", + "# # Example: Modify k-grids\n", "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", "#\n", "# new_context = PointsGridDataProvider(dimensions=KGRID, isEdited=True).yield_data()\n", @@ -342,14 +330,14 @@ "# unit = subworkflow.get_unit_by_name(name=\"pw_scf\") # Adjust unit name as needed\n", "# unit.add_context(new_context)\n", "# subworkflow.set_unit(unit)\n", - "\n", - "# Preview modified workflow\n", + "#\n", + "# # Preview modified workflow\n", "# visualize_workflow(workflow)\n" ] }, { "cell_type": "markdown", - "id": "25", + "id": "24", "metadata": {}, "source": [ "### 4.4. Save workflow to collection" @@ -358,7 +346,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -380,7 +368,7 @@ }, { "cell_type": "markdown", - "id": "27", + "id": "26", "metadata": {}, "source": [ "## 5. Create the compute configuration\n", @@ -390,7 +378,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -400,7 +388,7 @@ }, { "cell_type": "markdown", - "id": "29", + "id": "28", "metadata": {}, "source": [ "### 5.2. Create compute configuration for the job\n" @@ -409,7 +397,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -428,7 +416,7 @@ }, { "cell_type": "markdown", - "id": "31", + "id": "30", "metadata": {}, "source": [ "## 6. Create the job with material and workflow configuration\n", @@ -438,7 +426,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -470,7 +458,7 @@ }, { "cell_type": "markdown", - "id": "33", + "id": "32", "metadata": {}, "source": [ "## 7. Submit the job and monitor the status" @@ -479,7 +467,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -489,14 +477,14 @@ }, { "cell_type": "markdown", - "id": "35", + "id": "34", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -507,7 +495,7 @@ }, { "cell_type": "markdown", - "id": "37", + "id": "36", "metadata": {}, "source": [ "## 8. Retrieve results" @@ -516,24 +504,16 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "37", "metadata": {}, "outputs": [], "source": [ "from mat3ra.prode import PropertyName\n", "from utils.visualize import visualize_properties\n", "\n", - "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy)\n", + "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy.value)\n", "visualize_properties(property_data, title=\"Total Energy\")\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 8c7ebc3e523c7036fd0ff53361967fe234e3a8b3 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 2 Mar 2026 19:24:45 -0800 Subject: [PATCH 10/19] update: pyproject --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6579d5f0..468069ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,10 @@ dependencies = [ "pymatgen==2024.4.13", "mat3ra-made>=2025.8.13.post0", "mat3ra-utils>=2024.6.11.post0", - "mat3ra-wode", + "mat3ra-wode @ git+https://github.com/Exabyte-io/wode.git@5ead5c3c8fbdd871bac56d0d072b352f78e75642", "mat3ra-prode", - "mat3ra-ide @ git+https://github.com/Exabyte-io/ide.git@15dbf8c8ea8284e3481b97ef6e65bfd6d3318352", - "mat3ra-api-client @ git+https://github.com/Exabyte-io/api-client.git@b30965041557d9e27fcfca1f1bd26726910a9e63", + "mat3ra-ide", + "mat3ra-api-client" ] [project.optional-dependencies] From b2af9df4c61fd6079d8b31397aa3b180ca75b838 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 5 Mar 2026 17:04:33 -0800 Subject: [PATCH 11/19] update: finalize nb --- .../workflows/total_energy.ipynb | 180 ++++++++++++------ 1 file changed, 124 insertions(+), 56 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 109da065..561e2172 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -17,7 +17,8 @@ "5. Configure workflow\n", "6. Set up compute resources\n", "7. Create and submit job\n", - "8. Retrieve and visualize results" + "8. Monitor execution\n", + "9. Retrieve and visualize results" ] }, { @@ -53,7 +54,7 @@ "id": "3", "metadata": {}, "source": [ - "### 1.2. Set parameters and configurations for the workflow and job" + "### 1.1. Set parameters and configurations for the workflow and job" ] }, { @@ -78,7 +79,7 @@ "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", "MY_WORKFLOW_NAME = \"Total Energy\"\n", "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", - "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", + "ADD_RELAXATION = True # Whether to add relaxation subworkflow\n", "SAVE_WF_TO_COLLECTION = True # If True, workflow is saved to collection\n", "\n", "# 5. Compute parameters\n", @@ -89,13 +90,21 @@ "# 6. Job parameters\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", "JOB_NAME = f\"Total Energy {timestamp}\"\n", - "POLL_INTERVAL = 30 # seconds\n" + "POLL_INTERVAL = 30 # seconds" ] }, { "cell_type": "markdown", "id": "5", "metadata": {}, + "source": [ + "### 1.2. API Configuration (optional, for local development)" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, "source": [ "## 2. Authenticate and initialize API client\n", "### 2.1. Authenticate" @@ -104,7 +113,7 @@ { "cell_type": "code", "execution_count": null, - "id": "6", + "id": "7", "metadata": {}, "outputs": [], "source": [ @@ -116,7 +125,7 @@ }, { "cell_type": "markdown", - "id": "7", + "id": "8", "metadata": {}, "source": [ "### 2.2. Initialize API Client" @@ -125,18 +134,19 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": {}, "outputs": [], "source": [ "from mat3ra.api_client import APIClient\n", "\n", - "client = APIClient.authenticate()" + "client = APIClient.authenticate()\n", + "client" ] }, { "cell_type": "markdown", - "id": "9", + "id": "10", "metadata": {}, "source": [ "### 2.3. Select account to work under" @@ -145,7 +155,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -155,7 +165,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -170,7 +180,7 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "13", "metadata": {}, "source": [ "### 2.4. Select project" @@ -179,7 +189,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -190,7 +200,7 @@ }, { "cell_type": "markdown", - "id": "14", + "id": "15", "metadata": {}, "source": [ "## 3. Create material\n", @@ -200,7 +210,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -217,7 +227,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "17", "metadata": {}, "source": [ "### 3.2. Save material to the platform" @@ -226,7 +236,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -239,7 +249,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "19", "metadata": {}, "source": [ "## 4. Create workflow and set its parameters\n", @@ -249,7 +259,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -263,7 +273,7 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "21", "metadata": {}, "source": [ "### 4.2. Create workflow from standard workflows and preview it" @@ -272,7 +282,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -289,25 +299,42 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "23", "metadata": {}, "source": [ - "### 4.3. Modify workflow (Optional)" + "### 4.3. Modify workflow (Optional)\n", + "#### 4.3.1. Add relaxation" ] }, { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [ - "# Example: Add relaxation subworkflow\n", "if ADD_RELAXATION:\n", - " workflow.add_relaxation()\n", - "\n", - "\n", + " workflow.add_relaxation()\n" + ] + }, + { + "cell_type": "markdown", + "id": "25", + "metadata": {}, + "source": [ + "#### 4.3.2. Modify model and method parameters (Optional)\n", + "Uncomment the code below and adjust selection of model parameters as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26", + "metadata": {}, + "outputs": [], + "source": [ "# # Example: Change model parameters\n", + "#\n", "# from mat3ra.mode import Model\n", "# from mat3ra.standata.model_tree import ModelTreeStandata\n", "#\n", @@ -321,23 +348,57 @@ "#\n", "# for subworkflow in workflow.subworkflows:\n", "# subworkflow.model = model\n", - "\n", - "# # Example: Modify k-grids\n", + "#\n", + "# if ADD_RELAXATION:\n", + "# # If relaxation subworkflow is added, set the same model for it\n", + "# workflow.relaxation_subworkflow.model = model\n", + "#\n", + "# # Preview modified workflow\n", + "# visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "27", + "metadata": {}, + "source": [ + "#### 4.3.3. Modify important settings\n", + "Set k-grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "28", + "metadata": {}, + "outputs": [], + "source": [ "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "# RELAXATION_KGRID = [2, 2, 2]\n", + "# SCF_KGRID = [4, 4, 4]\n", + "# new_context_relax = PointsGridDataProvider(dimensions=RELAXATION_KGRID, isEdited=True).yield_data() if ADD_RELAXATION else None\n", + "# new_context_scf = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).yield_data()\n", + "#\n", + "# if ADD_RELAXATION:\n", + "# relaxation_subworkflow = workflow.relaxation_subworkflow\n", + "# unit_to_modify_relax = relaxation_subworkflow.get_unit_by_name(name_regex=\"relax\")\n", + "# unit_to_modify_relax.add_context(new_context_relax)\n", + "# # Set the modified unit back to the workflow\n", + "# # Option 1: direct set by unit object, replacing the existing one\n", + "# relaxation_subworkflow.set_unit(unit_to_modify_relax)\n", "#\n", - "# new_context = PointsGridDataProvider(dimensions=KGRID, isEdited=True).yield_data()\n", - "# subworkflow = workflow.subworkflows[0]\n", - "# unit = subworkflow.get_unit_by_name(name=\"pw_scf\") # Adjust unit name as needed\n", - "# unit.add_context(new_context)\n", - "# subworkflow.set_unit(unit)\n", + "# band_gap_subworkflow = workflow.subworkflows[1 if ADD_RELAXATION else 0]\n", + "# unit_to_modify_scf = band_gap_subworkflow.get_unit_by_name(name=\"pw_scf\")\n", + "# unit_to_modify_scf.add_context(new_context_scf)\n", + "# band_gap_subworkflow.set_unit(unit_to_modify_scf)\n", "#\n", "# # Preview modified workflow\n", - "# visualize_workflow(workflow)\n" + "# visualize_workflow(workflow)" ] }, { "cell_type": "markdown", - "id": "24", + "id": "29", "metadata": {}, "source": [ "### 4.4. Save workflow to collection" @@ -346,17 +407,16 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "30", "metadata": {}, "outputs": [], "source": [ "from utils.generic import dict_to_namespace\n", + "from utils.api import get_or_create_workflow\n", "\n", "workflow_id_or_dict = None\n", "\n", "if SAVE_WF_TO_COLLECTION:\n", - " from utils.api import get_or_create_workflow\n", - "\n", " saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", " saved_workflow = dict_to_namespace(saved_workflow_response)\n", " workflow_id_or_dict = saved_workflow._id\n", @@ -368,7 +428,7 @@ }, { "cell_type": "markdown", - "id": "26", + "id": "31", "metadata": {}, "source": [ "## 5. Create the compute configuration\n", @@ -378,17 +438,17 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "32", "metadata": {}, "outputs": [], "source": [ "clusters = client.clusters.list()\n", - "print(f\"Available clusters: {[c for c in clusters]}\")" + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" ] }, { "cell_type": "markdown", - "id": "28", + "id": "33", "metadata": {}, "source": [ "### 5.2. Create compute configuration for the job\n" @@ -397,7 +457,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -411,12 +471,12 @@ " queue=QUEUE_NAME,\n", " ppn=PPN\n", ")\n", - "compute.to_dict()" + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" ] }, { "cell_type": "markdown", - "id": "30", + "id": "35", "metadata": {}, "source": [ "## 6. Create the job with material and workflow configuration\n", @@ -426,7 +486,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -458,7 +518,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "37", "metadata": {}, "source": [ "## 7. Submit the job and monitor the status" @@ -467,7 +527,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -477,25 +537,25 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "39", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "40", "metadata": {}, "outputs": [], "source": [ - "from utils.api import wait_for_jobs_to_finish\n", + "from utils.api import wait_for_jobs_to_finish_async\n", "\n", - "wait_for_jobs_to_finish(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" + "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" ] }, { "cell_type": "markdown", - "id": "36", + "id": "41", "metadata": {}, "source": [ "## 8. Retrieve results" @@ -504,7 +564,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "42", "metadata": {}, "outputs": [], "source": [ @@ -514,6 +574,14 @@ "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy.value)\n", "visualize_properties(property_data, title=\"Total Energy\")\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From deefd614a9ff73237047e3a6e634d6146fad8b3a Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 5 Mar 2026 17:05:08 -0800 Subject: [PATCH 12/19] update: finalize nb 2 --- other/materials_designer/workflows/total_energy.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 561e2172..3c941510 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -79,7 +79,8 @@ "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", "MY_WORKFLOW_NAME = \"Total Energy\"\n", "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", - "ADD_RELAXATION = True # Whether to add relaxation subworkflow\n", + "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", + "\n", "SAVE_WF_TO_COLLECTION = True # If True, workflow is saved to collection\n", "\n", "# 5. Compute parameters\n", From e5ac9d465836ba8216b3ec1300bb80e12770013c Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 13:09:36 -0700 Subject: [PATCH 13/19] update: cleanup --- .../workflows/total_energy.ipynb | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 3c941510..ab327688 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -14,11 +14,11 @@ "2. Authenticate and initialize API client\n", "3. Select account and project\n", "4. Load and save materials\n", + "Materials should be saved to \"../uploads\" folder in Mat3ra JSON format and referenced by name in the MATERIAL_NAME variable. If material with the specified name is not found in the folder, attempt will be made to load it from Standata as a fallback.\n", "5. Configure workflow\n", "6. Set up compute resources\n", "7. Create and submit job\n", - "8. Monitor execution\n", - "9. Retrieve and visualize results" + "8. Retrieve and visualize results" ] }, { @@ -54,7 +54,7 @@ "id": "3", "metadata": {}, "source": [ - "### 1.1. Set parameters and configurations for the workflow and job" + "### 1.2. Set parameters and configurations for the workflow and job" ] }, { @@ -81,8 +81,6 @@ "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", "\n", - "SAVE_WF_TO_COLLECTION = True # If True, workflow is saved to collection\n", - "\n", "# 5. Compute parameters\n", "CLUSTER_NAME = None # specify i.e. \"cluster-001\" to use that cluster\n", "QUEUE_NAME = QueueName.D\n", @@ -99,16 +97,24 @@ "id": "5", "metadata": {}, "source": [ - "### 1.2. API Configuration (optional, for local development)" + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "id": "6", "metadata": {}, + "outputs": [], "source": [ - "## 2. Authenticate and initialize API client\n", - "### 2.1. Authenticate" + "import os\n", + "\n", + "os.environ[\"API_HOST\"] = \"localhost\"\n", + "os.environ[\"API_PORT\"] = \"3000\"\n", + "os.environ[\"API_SECURE\"] = \"false\"\n", + "\n", + "# os.environ[\"OIDC_ACCESS_TOKEN\"] = \"uHfQSwBAkMUJumJGYhaYm__9oDonUk6NhAiA0knJ661\"" ] }, { @@ -583,6 +589,14 @@ "metadata": {}, "outputs": [], "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 6534f2091ee54ee88bcc1a112f10cfebab6bdf34 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 13:49:46 -0700 Subject: [PATCH 14/19] update: cleanup --- .../workflows/total_energy.ipynb | 1226 ++++++++--------- utils/api.py | 11 - 2 files changed, 604 insertions(+), 633 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index ab327688..f6d92003 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -1,623 +1,605 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Total Energy\n", - "\n", - "This notebook demonstrates how to run a workflow calculation using the Mat3ra API.\n", - "\n", - "## Process Overview\n", - "1. Set up environment and parameters\n", - "2. Authenticate and initialize API client\n", - "3. Select account and project\n", - "4. Load and save materials\n", - "Materials should be saved to \"../uploads\" folder in Mat3ra JSON format and referenced by name in the MATERIAL_NAME variable. If material with the specified name is not found in the folder, attempt will be made to load it from Standata as a fallback.\n", - "5. Configure workflow\n", - "6. Set up compute resources\n", - "7. Create and submit job\n", - "8. Retrieve and visualize results" - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "## 1. Set up the environment and parameters\n", - "### 1.1. Install packages (JupyterLite)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "if sys.platform == \"emscripten\":\n", - " import micropip\n", - "\n", - " await micropip.install(\"mat3ra-api-examples\", deps=False)\n", - " await micropip.install(\"mat3ra-utils\")\n", - " from mat3ra.utils.jupyterlite.packages import install_packages\n", - "\n", - " await install_packages(\"api_examples\")\n" - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "### 1.2. Set parameters and configurations for the workflow and job" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "from mat3ra.ide.compute import QueueName\n", - "\n", - "# 2. Auth and organization parameters\n", - "# Set organization name to use it as the owner, otherwise your personal account is used\n", - "ORGANIZATION_NAME = None\n", - "\n", - "# 3. Material parameters\n", - "FOLDER = \"../uploads\"\n", - "MATERIAL_NAME = \"Silicon\" # Name of the material to load from local file or Standata\n", - "\n", - "# 4. Workflow parameters\n", - "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", - "MY_WORKFLOW_NAME = \"Total Energy\"\n", - "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", - "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", - "\n", - "# 5. Compute parameters\n", - "CLUSTER_NAME = None # specify i.e. \"cluster-001\" to use that cluster\n", - "QUEUE_NAME = QueueName.D\n", - "PPN = 1\n", - "\n", - "# 6. Job parameters\n", - "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", - "JOB_NAME = f\"Total Energy {timestamp}\"\n", - "POLL_INTERVAL = 30 # seconds" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "## 2. Authenticate and initialize API client\n", - "### 2.1. Authenticate" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "os.environ[\"API_HOST\"] = \"localhost\"\n", - "os.environ[\"API_PORT\"] = \"3000\"\n", - "os.environ[\"API_SECURE\"] = \"false\"\n", - "\n", - "# os.environ[\"OIDC_ACCESS_TOKEN\"] = \"uHfQSwBAkMUJumJGYhaYm__9oDonUk6NhAiA0knJ661\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "from utils.auth import authenticate\n", - "\n", - "# Authenticate in the browser and have credentials stored in environment variables\n", - "await authenticate()" - ] - }, - { - "cell_type": "markdown", - "id": "8", - "metadata": {}, - "source": [ - "### 2.2. Initialize API Client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.api_client import APIClient\n", - "\n", - "client = APIClient.authenticate()\n", - "client" - ] - }, - { - "cell_type": "markdown", - "id": "10", - "metadata": {}, - "source": [ - "### 2.3. Select account to work under" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "client.list_accounts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "selected_account = client.my_account\n", - "\n", - "if ORGANIZATION_NAME:\n", - " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", - "\n", - "ACCOUNT_ID = selected_account.id\n", - "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" - ] - }, - { - "cell_type": "markdown", - "id": "13", - "metadata": {}, - "source": [ - "### 2.4. Select project" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14", - "metadata": {}, - "outputs": [], - "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" - ] - }, - { - "cell_type": "markdown", - "id": "15", - "metadata": {}, - "source": [ - "## 3. Create material\n", - "### 3.1. Load material from local file (or Standata)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "16", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.made.material import Material\n", - "from mat3ra.standata.materials import Materials\n", - "from utils.visualize import visualize_materials as visualize\n", - "from utils.jupyterlite import load_material_from_folder\n", - "\n", - "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", - " Materials.get_by_name_first_match(MATERIAL_NAME))\n", - "\n", - "visualize(material)" - ] - }, - { - "cell_type": "markdown", - "id": "17", - "metadata": {}, - "source": [ - "### 3.2. Save material to the platform" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "18", - "metadata": {}, - "outputs": [], - "source": [ - "from utils.api import get_or_create_material\n", - "from utils.generic import dict_to_namespace\n", - "\n", - "saved_material_response = get_or_create_material(client.materials, material, ACCOUNT_ID)\n", - "saved_material = dict_to_namespace(saved_material_response)" - ] - }, - { - "cell_type": "markdown", - "id": "19", - "metadata": {}, - "source": [ - "## 4. Create workflow and set its parameters\n", - "### 4.1. Get list of applications and select one" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "20", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.standata.applications import ApplicationStandata\n", - "from mat3ra.ade.application import Application\n", - "\n", - "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", - "app = Application(**app_config)\n", - "print(f\"Using application: {app.name}\")" - ] - }, - { - "cell_type": "markdown", - "id": "21", - "metadata": {}, - "source": [ - "### 4.2. Create workflow from standard workflows and preview it" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "22", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.standata.workflows import WorkflowStandata\n", - "from mat3ra.wode.workflows import Workflow\n", - "from utils.visualize import visualize_workflow\n", - "\n", - "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", - "workflow = Workflow.create(workflow_config)\n", - "workflow.name = MY_WORKFLOW_NAME\n", - "\n", - "visualize_workflow(workflow)" - ] - }, - { - "cell_type": "markdown", - "id": "23", - "metadata": {}, - "source": [ - "### 4.3. Modify workflow (Optional)\n", - "#### 4.3.1. Add relaxation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24", - "metadata": {}, - "outputs": [], - "source": [ - "if ADD_RELAXATION:\n", - " workflow.add_relaxation()\n" - ] - }, - { - "cell_type": "markdown", - "id": "25", - "metadata": {}, - "source": [ - "#### 4.3.2. Modify model and method parameters (Optional)\n", - "Uncomment the code below and adjust selection of model parameters as needed." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26", - "metadata": {}, - "outputs": [], - "source": [ - "# # Example: Change model parameters\n", - "#\n", - "# from mat3ra.mode import Model\n", - "# from mat3ra.standata.model_tree import ModelTreeStandata\n", - "#\n", - "# model_config = ModelTreeStandata.get_model_by_parameters(\n", - "# type=\"dft\",\n", - "# subtype=\"gga\",\n", - "# functional=\"pbe\"\n", - "# )\n", - "# model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"us\"}\n", - "# model = Model.create(model_config)\n", - "#\n", - "# for subworkflow in workflow.subworkflows:\n", - "# subworkflow.model = model\n", - "#\n", - "# if ADD_RELAXATION:\n", - "# # If relaxation subworkflow is added, set the same model for it\n", - "# workflow.relaxation_subworkflow.model = model\n", - "#\n", - "# # Preview modified workflow\n", - "# visualize_workflow(workflow)" - ] - }, - { - "cell_type": "markdown", - "id": "27", - "metadata": {}, - "source": [ - "#### 4.3.3. Modify important settings\n", - "Set k-grid." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "28", - "metadata": {}, - "outputs": [], - "source": [ - "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", - "# RELAXATION_KGRID = [2, 2, 2]\n", - "# SCF_KGRID = [4, 4, 4]\n", - "# new_context_relax = PointsGridDataProvider(dimensions=RELAXATION_KGRID, isEdited=True).yield_data() if ADD_RELAXATION else None\n", - "# new_context_scf = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).yield_data()\n", - "#\n", - "# if ADD_RELAXATION:\n", - "# relaxation_subworkflow = workflow.relaxation_subworkflow\n", - "# unit_to_modify_relax = relaxation_subworkflow.get_unit_by_name(name_regex=\"relax\")\n", - "# unit_to_modify_relax.add_context(new_context_relax)\n", - "# # Set the modified unit back to the workflow\n", - "# # Option 1: direct set by unit object, replacing the existing one\n", - "# relaxation_subworkflow.set_unit(unit_to_modify_relax)\n", - "#\n", - "# band_gap_subworkflow = workflow.subworkflows[1 if ADD_RELAXATION else 0]\n", - "# unit_to_modify_scf = band_gap_subworkflow.get_unit_by_name(name=\"pw_scf\")\n", - "# unit_to_modify_scf.add_context(new_context_scf)\n", - "# band_gap_subworkflow.set_unit(unit_to_modify_scf)\n", - "#\n", - "# # Preview modified workflow\n", - "# visualize_workflow(workflow)" - ] - }, - { - "cell_type": "markdown", - "id": "29", - "metadata": {}, - "source": [ - "### 4.4. Save workflow to collection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "30", - "metadata": {}, - "outputs": [], - "source": [ - "from utils.generic import dict_to_namespace\n", - "from utils.api import get_or_create_workflow\n", - "\n", - "workflow_id_or_dict = None\n", - "\n", - "if SAVE_WF_TO_COLLECTION:\n", - " saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", - " saved_workflow = dict_to_namespace(saved_workflow_response)\n", - " workflow_id_or_dict = saved_workflow._id\n", - " print(f\"Workflow ID: {saved_workflow._id}\")\n", - "else:\n", - " workflow_id_or_dict = workflow.to_dict()\n", - " print(\"Workflow will be embedded into job (not saved to collection)\")" - ] - }, - { - "cell_type": "markdown", - "id": "31", - "metadata": {}, - "source": [ - "## 5. Create the compute configuration\n", - "### 5.1. Get list of clusters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "32", - "metadata": {}, - "outputs": [], - "source": [ - "clusters = client.clusters.list()\n", - "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "33", - "metadata": {}, - "source": [ - "### 5.2. Create compute configuration for the job\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "34", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.ide.compute import Compute\n", - "\n", - "# Select first available cluster or use specified name\n", - "cluster = next((c for c in clusters if c[\"hostname\"] == CLUSTER_NAME), clusters[0] if clusters else None)\n", - "\n", - "compute = Compute(\n", - " cluster=cluster,\n", - " queue=QUEUE_NAME,\n", - " ppn=PPN\n", - ")\n", - "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" - ] - }, - { - "cell_type": "markdown", - "id": "35", - "metadata": {}, - "source": [ - "## 6. Create the job with material and workflow configuration\n", - "### 6.1. Create job" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "from utils.api import create_job\n", - "from utils.visualize import display_JSON\n", - "\n", - "print(f\"Material: {saved_material._id}\")\n", - "print(f\"Workflow: {workflow_id_or_dict if SAVE_WF_TO_COLLECTION else '(embedded)'}\")\n", - "print(f\"Project: {project_id}\")\n", - "\n", - "job_response = create_job(\n", - " jobs_endpoint=client.jobs,\n", - " materials=[vars(saved_material)],\n", - " workflow_id_or_dict=workflow_id_or_dict,\n", - " project_id=project_id,\n", - " owner_id=ACCOUNT_ID,\n", - " prefix=JOB_NAME,\n", - " compute=compute.to_dict(),\n", - " save_to_collection=SAVE_WF_TO_COLLECTION,\n", - ")\n", - "\n", - "job_dict = job_response[0]\n", - "job = dict_to_namespace(job_dict)\n", - "job_id = job._id\n", - "print(\"✅ Job created successfully!\")\n", - "print(f\"Job ID: {job_id}\")\n", - "display_JSON(job_response)" - ] - }, - { - "cell_type": "markdown", - "id": "37", - "metadata": {}, - "source": [ - "## 7. Submit the job and monitor the status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "client.jobs.submit(job_id)\n", - "print(f\"✅ Job {job_id} submitted successfully!\")" - ] - }, - { - "cell_type": "markdown", - "id": "39", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40", - "metadata": {}, - "outputs": [], - "source": [ - "from utils.api import wait_for_jobs_to_finish_async\n", - "\n", - "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" - ] - }, - { - "cell_type": "markdown", - "id": "41", - "metadata": {}, - "source": [ - "## 8. Retrieve results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "42", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.prode import PropertyName\n", - "from utils.visualize import visualize_properties\n", - "\n", - "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy.value)\n", - "visualize_properties(property_data, title=\"Total Energy\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "43", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "44", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Total Energy\n", + "\n", + "This notebook demonstrates how to run a total energy workflow calculation using the Mat3ra API.\n", + "\n", + "## Process Overview\n", + "\n", + "### 1. Set up environment and parameters\n", + "Install required packages (JupyterLite only) and configure parameters for the material, workflow, compute resources, and job.\n", + "\n", + "### 2. Authenticate and initialize API client\n", + "Authenticate via browser, initialize the API client, and select the account and project to work under.\n", + "\n", + "### 3. Load and save material\n", + "Load material from the `../uploads` folder in Mat3ra JSON format, referenced by `MATERIAL_NAME`. If not found locally, the material is loaded from Standata as a fallback. The material is then saved to the platform.\n", + "\n", + "### 4. Create workflow and set its parameters\n", + "Select an application (e.g., Quantum ESPRESSO, VASP), load the total energy workflow from Standata, and optionally modify it — add relaxation, adjust model/method parameters, or set k-grid.\n", + "\n", + "### 5. Set up compute resources\n", + "Select a compute cluster and configure queue, number of processors, and job name.\n", + "\n", + "### 6. Create and submit job\n", + "Assemble the job from the material, workflow, and compute configuration, then submit it to the platform.\n", + "\n", + "### 7. Retrieve and visualize results\n", + "Poll for job completion and display the total energy result." + ], + "id": "0" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)" + ], + "id": "1" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import sys\n", + "\n", + "if sys.platform == \"emscripten\":\n", + " import micropip\n", + "\n", + " await micropip.install(\"mat3ra-api-examples\", deps=False)\n", + " await micropip.install(\"mat3ra-utils\")\n", + " from mat3ra.utils.jupyterlite.packages import install_packages\n", + "\n", + " await install_packages(\"api_examples\")\n" + ], + "execution_count": null, + "outputs": [], + "id": "2" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2. Set parameters and configurations for the workflow and job" + ], + "id": "3" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from datetime import datetime\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAME = \"Silicon\" # Name of the material to load from local file or Standata\n", + "\n", + "# 4. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", + "MY_WORKFLOW_NAME = \"Total Energy\"\n", + "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", + "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", + "\n", + "# 5. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 6. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "JOB_NAME = f\"{MY_WORKFLOW_NAME} {timestamp}\"\n", + "POLL_INTERVAL = 30 # seconds" + ], + "execution_count": null, + "outputs": [], + "id": "4" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate" + ], + "id": "6" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from utils.auth import authenticate\n", + "\n", + "# Authenticate in the browser and have credentials stored in environment variables\n", + "await authenticate()" + ], + "execution_count": null, + "outputs": [], + "id": "7" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2. Initialize API Client" + ], + "id": "8" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client" + ], + "execution_count": null, + "outputs": [], + "id": "9" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under" + ], + "id": "10" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "client.list_accounts()" + ], + "execution_count": null, + "outputs": [], + "id": "11" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" + ], + "execution_count": null, + "outputs": [], + "id": "12" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.4. Select project" + ], + "id": "13" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + ], + "execution_count": null, + "outputs": [], + "id": "14" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Create material\n", + "### 3.1. Load material from local file (or Standata)" + ], + "id": "15" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from utils.visualize import visualize_materials as visualize\n", + "from utils.jupyterlite import load_material_from_folder\n", + "\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", + " Materials.get_by_name_first_match(MATERIAL_NAME))\n", + "\n", + "visualize(material)" + ], + "execution_count": null, + "outputs": [], + "id": "16" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 3.2. Save material to the platform" + ], + "id": "17" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from utils.api import get_or_create_material\n", + "from utils.generic import dict_to_namespace\n", + "\n", + "saved_material_response = get_or_create_material(client.materials, material, ACCOUNT_ID)\n", + "saved_material = dict_to_namespace(saved_material_response)" + ], + "execution_count": null, + "outputs": [], + "id": "18" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Create workflow and set its parameters\n", + "### 4.1. Get list of applications and select one" + ], + "id": "19" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from mat3ra.standata.applications import ApplicationStandata\n", + "from mat3ra.ade.application import Application\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "print(f\"Using application: {app.name}\")" + ], + "execution_count": null, + "outputs": [], + "id": "20" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.2. Create workflow from standard workflows and preview it" + ], + "id": "21" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "from utils.visualize import visualize_workflow\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "workflow = Workflow.create(workflow_config)\n", + "workflow.name = MY_WORKFLOW_NAME\n", + "\n", + "visualize_workflow(workflow)" + ], + "execution_count": null, + "outputs": [], + "id": "22" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.3. Modify workflow (Optional)\n", + "#### 4.3.1. Add relaxation" + ], + "id": "23" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "if ADD_RELAXATION:\n", + " workflow.add_relaxation()\n" + ], + "execution_count": null, + "outputs": [], + "id": "24" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.3.2. Modify model and method parameters (Optional)\n", + "Uncomment the code below and adjust selection of model parameters as needed." + ], + "id": "25" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# # Example: Change model parameters\n", + "#\n", + "# from mat3ra.mode import Model\n", + "# from mat3ra.standata.model_tree import ModelTreeStandata\n", + "#\n", + "# model_config = ModelTreeStandata.get_model_by_parameters(\n", + "# type=\"dft\",\n", + "# subtype=\"gga\",\n", + "# functional=\"pbe\"\n", + "# )\n", + "# model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"us\"}\n", + "# model = Model.create(model_config)\n", + "#\n", + "# for subworkflow in workflow.subworkflows:\n", + "# subworkflow.model = model\n", + "#\n", + "# if ADD_RELAXATION:\n", + "# # If relaxation subworkflow is added, set the same model for it\n", + "# workflow.relaxation_subworkflow.model = model\n", + "#\n", + "# # Preview modified workflow\n", + "# visualize_workflow(workflow)" + ], + "execution_count": null, + "outputs": [], + "id": "26" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4.3.3. Modify important settings\n", + "Set k-grid." + ], + "id": "27" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "# RELAXATION_KGRID = [2, 2, 2]\n", + "# SCF_KGRID = [4, 4, 4]\n", + "# new_context_relax = PointsGridDataProvider(dimensions=RELAXATION_KGRID, isEdited=True).yield_data() if ADD_RELAXATION else None\n", + "# new_context_scf = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).yield_data()\n", + "#\n", + "# if ADD_RELAXATION:\n", + "# relaxation_subworkflow = workflow.relaxation_subworkflow\n", + "# unit_to_modify_relax = relaxation_subworkflow.get_unit_by_name(name_regex=\"relax\")\n", + "# unit_to_modify_relax.add_context(new_context_relax)\n", + "# # Set the modified unit back to the workflow\n", + "# # Option 1: direct set by unit object, replacing the existing one\n", + "# relaxation_subworkflow.set_unit(unit_to_modify_relax)\n", + "#\n", + "# band_gap_subworkflow = workflow.subworkflows[1 if ADD_RELAXATION else 0]\n", + "# unit_to_modify_scf = band_gap_subworkflow.get_unit_by_name(name=\"pw_scf\")\n", + "# unit_to_modify_scf.add_context(new_context_scf)\n", + "# band_gap_subworkflow.set_unit(unit_to_modify_scf)\n", + "#\n", + "# # Preview modified workflow\n", + "# visualize_workflow(workflow)" + ], + "execution_count": null, + "outputs": [], + "id": "28" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 4.4. Save workflow to collection" + ], + "id": "29" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from utils.generic import dict_to_namespace\n", + "from utils.api import get_or_create_workflow\n", + "\n", + "workflow_id_or_dict = None\n", + "\n", + "saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", + "saved_workflow = dict_to_namespace(saved_workflow_response)\n", + "print(f\"Workflow ID: {saved_workflow._id}\")" + ], + "execution_count": null, + "outputs": [], + "id": "30" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Create the compute configuration\n", + "### 5.1. Get list of clusters" + ], + "id": "31" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" + ], + "execution_count": null, + "outputs": [], + "id": "32" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 5.2. Create compute configuration for the job\n" + ], + "id": "33" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "# Select first available cluster or use specified name\n", + "cluster = next((c for c in clusters if CLUSTER_NAME and CLUSTER_NAME in c[\"hostname\"]), clusters[0] if clusters else None)\n", + "\n", + "compute = Compute(\n", + " cluster=cluster,\n", + " queue=QUEUE_NAME,\n", + " ppn=PPN\n", + ")\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + ], + "execution_count": null, + "outputs": [], + "id": "34" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Create the job with material and workflow configuration\n", + "### 6.1. Create job" + ], + "id": "35" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from utils.api import create_job\n", + "from utils.visualize import display_JSON\n", + "\n", + "print(f\"Material: {saved_material._id}\")\n", + "print(f\"Workflow: {saved_workflow._id}\")\n", + "print(f\"Project: {project_id}\")\n", + "\n", + "job_response = create_job(\n", + " jobs_endpoint=client.jobs,\n", + " materials=[vars(saved_material)],\n", + " workflow_id_or_dict=workflow_id_or_dict,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=JOB_NAME,\n", + " compute=compute.to_dict(),\n", + ")\n", + "\n", + "job_dict = job_response[0]\n", + "job = dict_to_namespace(job_dict)\n", + "job_id = job._id\n", + "print(\"✅ Job created successfully!\")\n", + "print(f\"Job ID: {job_id}\")\n", + "display_JSON(job_response)" + ], + "execution_count": null, + "outputs": [], + "id": "36" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Submit the job and monitor the status" + ], + "id": "37" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "client.jobs.submit(job_id)\n", + "print(f\"✅ Job {job_id} submitted successfully!\")" + ], + "execution_count": null, + "outputs": [], + "id": "38" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [], + "id": "39" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from utils.api import wait_for_jobs_to_finish_async\n", + "\n", + "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" + ], + "execution_count": null, + "outputs": [], + "id": "40" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 8. Retrieve results" + ], + "id": "41" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "from mat3ra.prode import PropertyName\n", + "from utils.visualize import visualize_properties\n", + "\n", + "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy.value)\n", + "visualize_properties(property_data, title=\"Total Energy\")\n" + ], + "execution_count": null, + "outputs": [], + "id": "42" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [], + "execution_count": null, + "outputs": [], + "id": "43" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/utils/api.py b/utils/api.py index 2ee049bb..bd76c9b0 100644 --- a/utils/api.py +++ b/utils/api.py @@ -181,7 +181,6 @@ def create_job( owner_id: str, prefix: str, compute: Optional[dict] = None, - save_to_collection: bool = True, ) -> List[dict]: """ Creates jobs for each material using either collection references or an embedded workflow. @@ -195,20 +194,10 @@ def create_job( owner_id (str): Account ID. prefix (str): Job name prefix. compute (dict, optional): Compute configuration dict. - save_to_collection (bool): If True, uses create_by_ids; otherwise embeds the workflow. Returns: list[dict]: List of created job dicts. """ - if save_to_collection: - return jobs_endpoint.create_by_ids( - materials=materials, - workflow_id=workflow_id_or_dict, - project_id=project_id, - prefix=prefix, - owner_id=owner_id, - compute=compute, - ) jobs = [] for material in materials: job_name = " ".join((prefix, material["formula"])) From 98d3875e01e830cea037d1b4bf5d056dcc680f25 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 14:09:08 -0700 Subject: [PATCH 15/19] update: better description --- .../workflows/total_energy.ipynb | 1203 ++++++++--------- 1 file changed, 599 insertions(+), 604 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index f6d92003..1b300ab8 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -1,605 +1,600 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Total Energy\n", - "\n", - "This notebook demonstrates how to run a total energy workflow calculation using the Mat3ra API.\n", - "\n", - "## Process Overview\n", - "\n", - "### 1. Set up environment and parameters\n", - "Install required packages (JupyterLite only) and configure parameters for the material, workflow, compute resources, and job.\n", - "\n", - "### 2. Authenticate and initialize API client\n", - "Authenticate via browser, initialize the API client, and select the account and project to work under.\n", - "\n", - "### 3. Load and save material\n", - "Load material from the `../uploads` folder in Mat3ra JSON format, referenced by `MATERIAL_NAME`. If not found locally, the material is loaded from Standata as a fallback. The material is then saved to the platform.\n", - "\n", - "### 4. Create workflow and set its parameters\n", - "Select an application (e.g., Quantum ESPRESSO, VASP), load the total energy workflow from Standata, and optionally modify it — add relaxation, adjust model/method parameters, or set k-grid.\n", - "\n", - "### 5. Set up compute resources\n", - "Select a compute cluster and configure queue, number of processors, and job name.\n", - "\n", - "### 6. Create and submit job\n", - "Assemble the job from the material, workflow, and compute configuration, then submit it to the platform.\n", - "\n", - "### 7. Retrieve and visualize results\n", - "Poll for job completion and display the total energy result." - ], - "id": "0" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Set up the environment and parameters\n", - "### 1.1. Install packages (JupyterLite)" - ], - "id": "1" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import sys\n", - "\n", - "if sys.platform == \"emscripten\":\n", - " import micropip\n", - "\n", - " await micropip.install(\"mat3ra-api-examples\", deps=False)\n", - " await micropip.install(\"mat3ra-utils\")\n", - " from mat3ra.utils.jupyterlite.packages import install_packages\n", - "\n", - " await install_packages(\"api_examples\")\n" - ], - "execution_count": null, - "outputs": [], - "id": "2" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 1.2. Set parameters and configurations for the workflow and job" - ], - "id": "3" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from datetime import datetime\n", - "from mat3ra.ide.compute import QueueName\n", - "\n", - "# 2. Auth and organization parameters\n", - "# Set organization name to use it as the owner, otherwise your personal account is used\n", - "ORGANIZATION_NAME = None\n", - "\n", - "# 3. Material parameters\n", - "FOLDER = \"../uploads\"\n", - "MATERIAL_NAME = \"Silicon\" # Name of the material to load from local file or Standata\n", - "\n", - "# 4. Workflow parameters\n", - "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", - "MY_WORKFLOW_NAME = \"Total Energy\"\n", - "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", - "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", - "\n", - "# 5. Compute parameters\n", - "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", - "QUEUE_NAME = QueueName.D\n", - "PPN = 1\n", - "\n", - "# 6. Job parameters\n", - "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", - "JOB_NAME = f\"{MY_WORKFLOW_NAME} {timestamp}\"\n", - "POLL_INTERVAL = 30 # seconds" - ], - "execution_count": null, - "outputs": [], - "id": "4" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Authenticate and initialize API client\n", - "### 2.1. Authenticate" - ], - "id": "6" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from utils.auth import authenticate\n", - "\n", - "# Authenticate in the browser and have credentials stored in environment variables\n", - "await authenticate()" - ], - "execution_count": null, - "outputs": [], - "id": "7" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.2. Initialize API Client" - ], - "id": "8" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from mat3ra.api_client import APIClient\n", - "\n", - "client = APIClient.authenticate()\n", - "client" - ], - "execution_count": null, - "outputs": [], - "id": "9" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.3. Select account to work under" - ], - "id": "10" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "client.list_accounts()" - ], - "execution_count": null, - "outputs": [], - "id": "11" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "selected_account = client.my_account\n", - "\n", - "if ORGANIZATION_NAME:\n", - " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", - "\n", - "ACCOUNT_ID = selected_account.id\n", - "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" - ], - "execution_count": null, - "outputs": [], - "id": "12" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 2.4. Select project" - ], - "id": "13" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" - ], - "execution_count": null, - "outputs": [], - "id": "14" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Create material\n", - "### 3.1. Load material from local file (or Standata)" - ], - "id": "15" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from mat3ra.made.material import Material\n", - "from mat3ra.standata.materials import Materials\n", - "from utils.visualize import visualize_materials as visualize\n", - "from utils.jupyterlite import load_material_from_folder\n", - "\n", - "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", - " Materials.get_by_name_first_match(MATERIAL_NAME))\n", - "\n", - "visualize(material)" - ], - "execution_count": null, - "outputs": [], - "id": "16" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 3.2. Save material to the platform" - ], - "id": "17" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from utils.api import get_or_create_material\n", - "from utils.generic import dict_to_namespace\n", - "\n", - "saved_material_response = get_or_create_material(client.materials, material, ACCOUNT_ID)\n", - "saved_material = dict_to_namespace(saved_material_response)" - ], - "execution_count": null, - "outputs": [], - "id": "18" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Create workflow and set its parameters\n", - "### 4.1. Get list of applications and select one" - ], - "id": "19" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from mat3ra.standata.applications import ApplicationStandata\n", - "from mat3ra.ade.application import Application\n", - "\n", - "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", - "app = Application(**app_config)\n", - "print(f\"Using application: {app.name}\")" - ], - "execution_count": null, - "outputs": [], - "id": "20" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.2. Create workflow from standard workflows and preview it" - ], - "id": "21" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from mat3ra.standata.workflows import WorkflowStandata\n", - "from mat3ra.wode.workflows import Workflow\n", - "from utils.visualize import visualize_workflow\n", - "\n", - "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", - "workflow = Workflow.create(workflow_config)\n", - "workflow.name = MY_WORKFLOW_NAME\n", - "\n", - "visualize_workflow(workflow)" - ], - "execution_count": null, - "outputs": [], - "id": "22" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.3. Modify workflow (Optional)\n", - "#### 4.3.1. Add relaxation" - ], - "id": "23" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "if ADD_RELAXATION:\n", - " workflow.add_relaxation()\n" - ], - "execution_count": null, - "outputs": [], - "id": "24" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 4.3.2. Modify model and method parameters (Optional)\n", - "Uncomment the code below and adjust selection of model parameters as needed." - ], - "id": "25" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# # Example: Change model parameters\n", - "#\n", - "# from mat3ra.mode import Model\n", - "# from mat3ra.standata.model_tree import ModelTreeStandata\n", - "#\n", - "# model_config = ModelTreeStandata.get_model_by_parameters(\n", - "# type=\"dft\",\n", - "# subtype=\"gga\",\n", - "# functional=\"pbe\"\n", - "# )\n", - "# model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"us\"}\n", - "# model = Model.create(model_config)\n", - "#\n", - "# for subworkflow in workflow.subworkflows:\n", - "# subworkflow.model = model\n", - "#\n", - "# if ADD_RELAXATION:\n", - "# # If relaxation subworkflow is added, set the same model for it\n", - "# workflow.relaxation_subworkflow.model = model\n", - "#\n", - "# # Preview modified workflow\n", - "# visualize_workflow(workflow)" - ], - "execution_count": null, - "outputs": [], - "id": "26" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 4.3.3. Modify important settings\n", - "Set k-grid." - ], - "id": "27" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", - "# RELAXATION_KGRID = [2, 2, 2]\n", - "# SCF_KGRID = [4, 4, 4]\n", - "# new_context_relax = PointsGridDataProvider(dimensions=RELAXATION_KGRID, isEdited=True).yield_data() if ADD_RELAXATION else None\n", - "# new_context_scf = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).yield_data()\n", - "#\n", - "# if ADD_RELAXATION:\n", - "# relaxation_subworkflow = workflow.relaxation_subworkflow\n", - "# unit_to_modify_relax = relaxation_subworkflow.get_unit_by_name(name_regex=\"relax\")\n", - "# unit_to_modify_relax.add_context(new_context_relax)\n", - "# # Set the modified unit back to the workflow\n", - "# # Option 1: direct set by unit object, replacing the existing one\n", - "# relaxation_subworkflow.set_unit(unit_to_modify_relax)\n", - "#\n", - "# band_gap_subworkflow = workflow.subworkflows[1 if ADD_RELAXATION else 0]\n", - "# unit_to_modify_scf = band_gap_subworkflow.get_unit_by_name(name=\"pw_scf\")\n", - "# unit_to_modify_scf.add_context(new_context_scf)\n", - "# band_gap_subworkflow.set_unit(unit_to_modify_scf)\n", - "#\n", - "# # Preview modified workflow\n", - "# visualize_workflow(workflow)" - ], - "execution_count": null, - "outputs": [], - "id": "28" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 4.4. Save workflow to collection" - ], - "id": "29" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from utils.generic import dict_to_namespace\n", - "from utils.api import get_or_create_workflow\n", - "\n", - "workflow_id_or_dict = None\n", - "\n", - "saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", - "saved_workflow = dict_to_namespace(saved_workflow_response)\n", - "print(f\"Workflow ID: {saved_workflow._id}\")" - ], - "execution_count": null, - "outputs": [], - "id": "30" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Create the compute configuration\n", - "### 5.1. Get list of clusters" - ], - "id": "31" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "clusters = client.clusters.list()\n", - "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" - ], - "execution_count": null, - "outputs": [], - "id": "32" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 5.2. Create compute configuration for the job\n" - ], - "id": "33" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from mat3ra.ide.compute import Compute\n", - "\n", - "# Select first available cluster or use specified name\n", - "cluster = next((c for c in clusters if CLUSTER_NAME and CLUSTER_NAME in c[\"hostname\"]), clusters[0] if clusters else None)\n", - "\n", - "compute = Compute(\n", - " cluster=cluster,\n", - " queue=QUEUE_NAME,\n", - " ppn=PPN\n", - ")\n", - "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" - ], - "execution_count": null, - "outputs": [], - "id": "34" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. Create the job with material and workflow configuration\n", - "### 6.1. Create job" - ], - "id": "35" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from utils.api import create_job\n", - "from utils.visualize import display_JSON\n", - "\n", - "print(f\"Material: {saved_material._id}\")\n", - "print(f\"Workflow: {saved_workflow._id}\")\n", - "print(f\"Project: {project_id}\")\n", - "\n", - "job_response = create_job(\n", - " jobs_endpoint=client.jobs,\n", - " materials=[vars(saved_material)],\n", - " workflow_id_or_dict=workflow_id_or_dict,\n", - " project_id=project_id,\n", - " owner_id=ACCOUNT_ID,\n", - " prefix=JOB_NAME,\n", - " compute=compute.to_dict(),\n", - ")\n", - "\n", - "job_dict = job_response[0]\n", - "job = dict_to_namespace(job_dict)\n", - "job_id = job._id\n", - "print(\"✅ Job created successfully!\")\n", - "print(f\"Job ID: {job_id}\")\n", - "display_JSON(job_response)" - ], - "execution_count": null, - "outputs": [], - "id": "36" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Submit the job and monitor the status" - ], - "id": "37" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "client.jobs.submit(job_id)\n", - "print(f\"✅ Job {job_id} submitted successfully!\")" - ], - "execution_count": null, - "outputs": [], - "id": "38" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [], - "id": "39" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from utils.api import wait_for_jobs_to_finish_async\n", - "\n", - "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" - ], - "execution_count": null, - "outputs": [], - "id": "40" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 8. Retrieve results" - ], - "id": "41" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from mat3ra.prode import PropertyName\n", - "from utils.visualize import visualize_properties\n", - "\n", - "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy.value)\n", - "visualize_properties(property_data, title=\"Total Energy\")\n" - ], - "execution_count": null, - "outputs": [], - "id": "42" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [], - "execution_count": null, - "outputs": [], - "id": "43" - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Total Energy\n", + "\n", + "Calculate the total energy of a material using a DFT workflow on the Mat3ra platform.\n", + "\n", + "

Usage

\n", + "\n", + "1. Set notebook parameters in cell 1.2. below (or use the default values).\n", + "1. Click \"Run\" > \"Run All\" to run all cells.\n", + "1. Wait for the job to complete (can take a few minutes depending on compute settings).\n", + "1. Scroll down to view the total energy result.\n", + "\n", + "## Summary\n", + "\n", + "1. Set up the environment and parameters: install packages (JupyterLite only) and configure parameters for material, workflow, compute resources, and job.\n", + "1. Authenticate and initialize API client: authenticate via browser, initialize the client, then select account and project.\n", + "1. Create material: materials are read from the `../uploads` folder — place files there manually or run a material creation notebook first. If the material is not found by name, Standata is used as a fallback. The material is then saved to the platform.\n", + "1. Create workflow and set its parameters: select application, load total energy workflow from Standata, optionally add relaxation or adjust model/method parameters, and save the workflow to the platform.\n", + "1. Configure compute: get list of clusters and create compute configuration with selected cluster, queue, and number of processors.\n", + "1. Create the job with material and workflow configuration: assemble the job from material, workflow, project, and compute configuration.\n", + "1. Submit the job and monitor the status: submit the job and wait for completion.\n", + "1. Retrieve results: get and display the properies." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "if sys.platform == \"emscripten\":\n", + " import micropip\n", + "\n", + " await micropip.install(\"mat3ra-api-examples\", deps=False)\n", + " await micropip.install(\"mat3ra-utils\")\n", + " from mat3ra.utils.jupyterlite.packages import install_packages\n", + "\n", + " await install_packages(\"api_examples\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.2. Set parameters and configurations for the workflow and job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAME = \"Silicon\" # Name of the material to load from local file or Standata\n", + "\n", + "# 4. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"total_energy.json\"\n", + "MY_WORKFLOW_NAME = \"Total Energy\"\n", + "APPLICATION_NAME = \"espresso\" # Specify application name (e.g., \"espresso\", \"vasp\", \"nwchem\")\n", + "ADD_RELAXATION = False # Whether to add relaxation subworkflow\n", + "\n", + "# 5. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 6. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "JOB_NAME = f\"{MY_WORKFLOW_NAME} {timestamp}\"\n", + "POLL_INTERVAL = 30 # seconds" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.auth import authenticate\n", + "\n", + "# Authenticate in the browser and have credentials stored in environment variables\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### 2.2. Initialize API Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "### 2.4. Select project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## 3. Create material\n", + "### 3.1. Load material from local file (or Standata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from utils.visualize import visualize_materials as visualize\n", + "from utils.jupyterlite import load_material_from_folder\n", + "\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", + " Materials.get_by_name_first_match(MATERIAL_NAME))\n", + "\n", + "visualize(material)" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### 3.2. Save material to the platform" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.api import get_or_create_material\n", + "from utils.generic import dict_to_namespace\n", + "\n", + "saved_material_response = get_or_create_material(client.materials, material, ACCOUNT_ID)\n", + "saved_material = dict_to_namespace(saved_material_response)" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "## 4. Create workflow and set its parameters\n", + "### 4.1. Get list of applications and select one" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.applications import ApplicationStandata\n", + "from mat3ra.ade.application import Application\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "print(f\"Using application: {app.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "### 4.2. Create workflow from standard workflows and preview it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "from utils.visualize import visualize_workflow\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "workflow = Workflow.create(workflow_config)\n", + "workflow.name = MY_WORKFLOW_NAME\n", + "\n", + "visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.3. Modify workflow (Optional)\n", + "#### 4.3.1. Add relaxation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "if ADD_RELAXATION:\n", + " workflow.add_relaxation()\n" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "#### 4.3.2. Modify model and method parameters (Optional)\n", + "Uncomment the code below and adjust selection of model parameters as needed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "# # Example: Change model parameters\n", + "#\n", + "# from mat3ra.mode import Model\n", + "# from mat3ra.standata.model_tree import ModelTreeStandata\n", + "#\n", + "# model_config = ModelTreeStandata.get_model_by_parameters(\n", + "# type=\"dft\",\n", + "# subtype=\"gga\",\n", + "# functional=\"pbe\"\n", + "# )\n", + "# model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"us\"}\n", + "# model = Model.create(model_config)\n", + "#\n", + "# for subworkflow in workflow.subworkflows:\n", + "# subworkflow.model = model\n", + "#\n", + "# if ADD_RELAXATION:\n", + "# # If relaxation subworkflow is added, set the same model for it\n", + "# workflow.relaxation_subworkflow.model = model\n", + "#\n", + "# # Preview modified workflow\n", + "# visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "#### 4.3.3. Modify important settings\n", + "Set k-grid." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "# from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "# RELAXATION_KGRID = [2, 2, 2]\n", + "# SCF_KGRID = [4, 4, 4]\n", + "# new_context_relax = PointsGridDataProvider(dimensions=RELAXATION_KGRID, isEdited=True).yield_data() if ADD_RELAXATION else None\n", + "# new_context_scf = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).yield_data()\n", + "#\n", + "# if ADD_RELAXATION:\n", + "# relaxation_subworkflow = workflow.relaxation_subworkflow\n", + "# unit_to_modify_relax = relaxation_subworkflow.get_unit_by_name(name_regex=\"relax\")\n", + "# unit_to_modify_relax.add_context(new_context_relax)\n", + "# # Set the modified unit back to the workflow\n", + "# # Option 1: direct set by unit object, replacing the existing one\n", + "# relaxation_subworkflow.set_unit(unit_to_modify_relax)\n", + "#\n", + "# band_gap_subworkflow = workflow.subworkflows[1 if ADD_RELAXATION else 0]\n", + "# unit_to_modify_scf = band_gap_subworkflow.get_unit_by_name(name=\"pw_scf\")\n", + "# unit_to_modify_scf.add_context(new_context_scf)\n", + "# band_gap_subworkflow.set_unit(unit_to_modify_scf)\n", + "#\n", + "# # Preview modified workflow\n", + "# visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "### 4.4. Save workflow to collection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.generic import dict_to_namespace\n", + "from utils.api import get_or_create_workflow\n", + "\n", + "workflow_id_or_dict = None\n", + "\n", + "saved_workflow_response = get_or_create_workflow(client.workflows, workflow, ACCOUNT_ID)\n", + "saved_workflow = dict_to_namespace(saved_workflow_response)\n", + "print(f\"Workflow ID: {saved_workflow._id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "## 5. Create the compute configuration\n", + "### 5.1. Get list of clusters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "### 5.2. Create compute configuration for the job\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "# Select first available cluster or use specified name\n", + "cluster = next((c for c in clusters if CLUSTER_NAME and CLUSTER_NAME in c[\"hostname\"]), clusters[0] if clusters else None)\n", + "\n", + "compute = Compute(\n", + " cluster=cluster,\n", + " queue=QUEUE_NAME,\n", + " ppn=PPN\n", + ")\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "## 6. Create the job with material and workflow configuration\n", + "### 6.1. Create job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.api import create_job\n", + "from utils.visualize import display_JSON\n", + "\n", + "print(f\"Material: {saved_material._id}\")\n", + "print(f\"Workflow: {saved_workflow._id}\")\n", + "print(f\"Project: {project_id}\")\n", + "\n", + "job_response = create_job(\n", + " jobs_endpoint=client.jobs,\n", + " materials=[vars(saved_material)],\n", + " workflow_id_or_dict=workflow_id_or_dict,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=JOB_NAME,\n", + " compute=compute.to_dict(),\n", + ")\n", + "\n", + "job_dict = job_response[0]\n", + "job = dict_to_namespace(job_dict)\n", + "job_id = job._id\n", + "print(\"✅ Job created successfully!\")\n", + "print(f\"Job ID: {job_id}\")\n", + "display_JSON(job_response)" + ] + }, + { + "cell_type": "markdown", + "id": "36", + "metadata": {}, + "source": [ + "## 7. Submit the job and monitor the status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "client.jobs.submit(job_id)\n", + "print(f\"✅ Job {job_id} submitted successfully!\")" + ] + }, + { + "cell_type": "markdown", + "id": "38", + "metadata": {}, + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39", + "metadata": {}, + "outputs": [], + "source": [ + "from utils.api import wait_for_jobs_to_finish_async\n", + "\n", + "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" + ] + }, + { + "cell_type": "markdown", + "id": "40", + "metadata": {}, + "source": [ + "## 8. Retrieve results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.prode import PropertyName\n", + "from utils.visualize import visualize_properties\n", + "\n", + "property_data = client.properties.get_for_job(job_id, property_name=PropertyName.scalar.total_energy.value)\n", + "visualize_properties(property_data, title=\"Total Energy\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From c576d2b1d5d104ada52c4cea6a1e462bb5e666b3 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 15:15:50 -0700 Subject: [PATCH 16/19] update: cleanup 2 --- .../workflows/total_energy.ipynb | 16 +++++----------- utils/api.py | 7 +++---- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 1b300ab8..c94b1759 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -493,7 +493,7 @@ "job_response = create_job(\n", " jobs_endpoint=client.jobs,\n", " materials=[vars(saved_material)],\n", - " workflow_id_or_dict=workflow_id_or_dict,\n", + " workflow_dict=workflow_id_or_dict,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=JOB_NAME,\n", @@ -527,16 +527,10 @@ "print(f\"✅ Job {job_id} submitted successfully!\")" ] }, - { - "cell_type": "markdown", - "id": "38", - "metadata": {}, - "source": [] - }, { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -547,7 +541,7 @@ }, { "cell_type": "markdown", - "id": "40", + "id": "39", "metadata": {}, "source": [ "## 8. Retrieve results" @@ -556,7 +550,7 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "40", "metadata": {}, "outputs": [], "source": [ @@ -570,7 +564,7 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "41", "metadata": {}, "outputs": [], "source": [] diff --git a/utils/api.py b/utils/api.py index bd76c9b0..9ae049fe 100644 --- a/utils/api.py +++ b/utils/api.py @@ -176,7 +176,7 @@ def get_or_create_workflow(endpoint: WorkflowEndpoints, workflow, owner_id: str) def create_job( jobs_endpoint: JobEndpoints, materials: List[dict], - workflow_id_or_dict, + workflow_dict, project_id: str, owner_id: str, prefix: str, @@ -188,8 +188,7 @@ def create_job( Args: jobs_endpoint (JobEndpoints): Job endpoint from the API client. materials (list[dict]): List of material dicts (must include _id and formula). - workflow_id_or_dict: Workflow _id (str) if save_to_collection=True, - or full workflow dict if save_to_collection=False. + workflow_dict: Workflow dictionary. project_id (str): Project ID. owner_id (str): Account ID. prefix (str): Job name prefix. @@ -201,7 +200,7 @@ def create_job( jobs = [] for material in materials: job_name = " ".join((prefix, material["formula"])) - embedded_workflow = {k: v for k, v in workflow_id_or_dict.items() if k != "_id"} + embedded_workflow = {k: v for k, v in workflow_dict.items() if k != "_id"} config = { "_project": {"_id": project_id}, "workflow": embedded_workflow, From 495589af077b3e96916fc334ecddca2b527d9fb1 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 15:40:32 -0700 Subject: [PATCH 17/19] update: wf dict --- other/materials_designer/workflows/total_energy.ipynb | 9 ++++++--- utils/api.py | 8 +++++++- utils/generic.py | 9 +++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index c94b1759..8ab723eb 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -456,8 +456,11 @@ "source": [ "from mat3ra.ide.compute import Compute\n", "\n", - "# Select first available cluster or use specified name\n", - "cluster = next((c for c in clusters if CLUSTER_NAME and CLUSTER_NAME in c[\"hostname\"]), clusters[0] if clusters else None)\n", + "# Select cluster: use specified name if provided, otherwise use first available\n", + "if CLUSTER_NAME:\n", + " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", + "else:\n", + " cluster = clusters[0]\n", "\n", "compute = Compute(\n", " cluster=cluster,\n", @@ -493,7 +496,7 @@ "job_response = create_job(\n", " jobs_endpoint=client.jobs,\n", " materials=[vars(saved_material)],\n", - " workflow_dict=workflow_id_or_dict,\n", + " workflow_dict=saved_workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=JOB_NAME,\n", diff --git a/utils/api.py b/utils/api.py index 9ae049fe..27ac27a1 100644 --- a/utils/api.py +++ b/utils/api.py @@ -3,6 +3,7 @@ import os import time import urllib.request +from types import SimpleNamespace from typing import List, Optional from mat3ra.api_client.endpoints.bank_workflows import BankWorkflowEndpoints @@ -12,6 +13,8 @@ from mat3ra.api_client.endpoints.workflows import WorkflowEndpoints from tabulate import tabulate +from utils.generic import namespace_to_dict + def save_files(job_id: str, job_endpoint: JobEndpoints, filename_on_cloud: str, filename_on_disk: str) -> None: """ @@ -188,7 +191,7 @@ def create_job( Args: jobs_endpoint (JobEndpoints): Job endpoint from the API client. materials (list[dict]): List of material dicts (must include _id and formula). - workflow_dict: Workflow dictionary. + workflow_dict: Workflow dictionary or namespace object. project_id (str): Project ID. owner_id (str): Account ID. prefix (str): Job name prefix. @@ -197,6 +200,9 @@ def create_job( Returns: list[dict]: List of created job dicts. """ + if isinstance(workflow_dict, SimpleNamespace): + workflow_dict = namespace_to_dict(workflow_dict) + jobs = [] for material in materials: job_name = " ".join((prefix, material["formula"])) diff --git a/utils/generic.py b/utils/generic.py index d9c86f17..02f0ffe1 100644 --- a/utils/generic.py +++ b/utils/generic.py @@ -38,3 +38,12 @@ def dict_to_namespace(obj): return [dict_to_namespace(item) for item in obj] else: return obj + + +def namespace_to_dict(obj): + if isinstance(obj, SimpleNamespace): + return {k: namespace_to_dict(v) for k, v in vars(obj).items()} + elif isinstance(obj, list): + return [namespace_to_dict(item) for item in obj] + else: + return obj From 3a4d3a4b9c404e39eb9c3699c77c4fbde9d3f983 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 15:53:31 -0700 Subject: [PATCH 18/19] chore: pyproject --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 468069ae..a3a7a41c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "pymatgen==2024.4.13", "mat3ra-made>=2025.8.13.post0", "mat3ra-utils>=2024.6.11.post0", - "mat3ra-wode @ git+https://github.com/Exabyte-io/wode.git@5ead5c3c8fbdd871bac56d0d072b352f78e75642", + "mat3ra-wode", "mat3ra-prode", "mat3ra-ide", "mat3ra-api-client" From d52ccfc57c7f9b9d7e1a693b241a39bec378fb12 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 9 Mar 2026 17:13:21 -0700 Subject: [PATCH 19/19] update: description --- other/materials_designer/workflows/total_energy.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 8ab723eb..d563286d 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -11,10 +11,10 @@ "\n", "

Usage

\n", "\n", - "1. Set notebook parameters in cell 1.2. below (or use the default values).\n", + "1. Set material and calculation parameters in cell 1.2. below (or use the default values).\n", "1. Click \"Run\" > \"Run All\" to run all cells.\n", - "1. Wait for the job to complete (can take a few minutes depending on compute settings).\n", - "1. Scroll down to view the total energy result.\n", + "1. Wait for the job to complete.\n", + "1. Scroll down to view the result.\n", "\n", "## Summary\n", "\n",