diff --git a/.github/workflows/tidy3d-python-client-tests.yml b/.github/workflows/tidy3d-python-client-tests.yml index 010e0fb1b6..eb08db9930 100644 --- a/.github/workflows/tidy3d-python-client-tests.yml +++ b/.github/workflows/tidy3d-python-client-tests.yml @@ -262,7 +262,7 @@ jobs: BRANCH_NAME="${STEPS_EXTRACT_BRANCH_NAME_OUTPUTS_BRANCH_NAME}" echo $BRANCH_NAME # Allow only Jira keys from known projects, even if the branch has an author prefix - ALLOWED_JIRA_PROJECTS=("FXC" "SCEM") + ALLOWED_JIRA_PROJECTS=("FXC" "SCEM" "SCRF") JIRA_PROJECT_PATTERN=$(IFS='|'; echo "${ALLOWED_JIRA_PROJECTS[*]}") JIRA_PATTERN="(${JIRA_PROJECT_PATTERN})-[0-9]+" diff --git a/docs/api/submit_simulations.rst b/docs/api/submit_simulations.rst index 4c08f6e9d7..795f045586 100644 --- a/docs/api/submit_simulations.rst +++ b/docs/api/submit_simulations.rst @@ -94,7 +94,6 @@ Information Containers :template: module.rst tidy3d.web.core.task_info.TaskInfo - tidy3d.web.core.task_info.TaskStatus Mode Solver Web API diff --git a/tests/test_plugins/test_array_factor.py b/tests/test_plugins/test_array_factor.py index 2ffe92afee..959e1dc7b0 100644 --- a/tests/test_plugins/test_array_factor.py +++ b/tests/test_plugins/test_array_factor.py @@ -363,16 +363,15 @@ def make_antenna_sim(): remove_dc_component=False, # Include DC component for more accuracy at low frequencies ) - sim_unit = list(modeler.sim_dict.values())[0] - - return sim_unit + return modeler def test_rectangular_array_calculator_array_make_antenna_array(): """Test automatic antenna array creation.""" freq0 = 10e9 wavelength0 = td.C_0 / 10e9 - sim_unit = make_antenna_sim() + modeler = make_antenna_sim() + sim_unit = list(modeler.sim_dict.values())[0] array_calculator = mw.RectangularAntennaArrayCalculator( array_size=(1, 2, 3), spacings=(0.5 * wavelength0, 0.6 * wavelength0, 0.4 * wavelength0), @@ -437,8 +436,9 @@ def test_rectangular_array_calculator_array_make_antenna_array(): assert len(sim_array.sources) == 6 # check that override_structures are duplicated - assert len(sim_unit.grid_spec.override_structures) == 2 - assert len(sim_array.grid_spec.override_structures) == 7 + # assert len(sim_unit.grid_spec.override_structures) == 2 + # assert len(sim_array.grid_spec.override_structures) == 7 + assert sim_unit.grid.boundaries == modeler.base_sim.grid.boundaries # check that phase shifts are applied correctly phases_expected = array_calculator._antenna_phases @@ -674,7 +674,8 @@ def test_rectangular_array_calculator_simulation_data_from_array_factor(): phase_shifts=(np.pi / 3, np.pi / 4, np.pi / 5), ) - sim_unit = make_antenna_sim() + modeler = make_antenna_sim() + sim_unit = list(modeler.sim_dict.values())[0] monitor = sim_unit.monitors[0] monitor_directivity = sim_unit.monitors[2] diff --git a/tidy3d/components/mode/mode_solver.py b/tidy3d/components/mode/mode_solver.py index 17a861dcbb..643676aeed 100644 --- a/tidy3d/components/mode/mode_solver.py +++ b/tidy3d/components/mode/mode_solver.py @@ -2655,7 +2655,7 @@ def _validate_modes_size(self) -> None: "frequencies or modes." ) - def validate_pre_upload(self, source_required: bool = True) -> None: + def validate_pre_upload(self) -> None: """Validate the fully initialized mode solver is ok for upload to our servers.""" self._validate_modes_size() diff --git a/tidy3d/components/mode/simulation.py b/tidy3d/components/mode/simulation.py index 2e836c55ec..e506706822 100644 --- a/tidy3d/components/mode/simulation.py +++ b/tidy3d/components/mode/simulation.py @@ -612,8 +612,8 @@ def plot_pml_mode_plane( """ return self._mode_solver.plot_pml(ax=ax) - def validate_pre_upload(self, source_required: bool = False) -> None: + def validate_pre_upload(self) -> None: super().validate_pre_upload() - self._mode_solver.validate_pre_upload(source_required=source_required) + self._mode_solver.validate_pre_upload() _boundaries_for_zero_dims = validate_boundaries_for_zero_dims(warn_on_change=False) diff --git a/tidy3d/components/tcad/mesher.py b/tidy3d/components/tcad/mesher.py index d35b465c9a..7e90e2461f 100644 --- a/tidy3d/components/tcad/mesher.py +++ b/tidy3d/components/tcad/mesher.py @@ -24,3 +24,6 @@ class VolumeMesher(Tidy3dBaseModel): def _get_simulation_types(self) -> list[TCADAnalysisTypes]: return [TCADAnalysisTypes.MESH] + + def validate_pre_upload(self): + return diff --git a/tidy3d/plugins/smatrix/component_modelers/base.py b/tidy3d/plugins/smatrix/component_modelers/base.py index 1d9715d98f..74c5afe1c0 100644 --- a/tidy3d/plugins/smatrix/component_modelers/base.py +++ b/tidy3d/plugins/smatrix/component_modelers/base.py @@ -343,5 +343,9 @@ def run( ) return data.smatrix() + def validate_pre_upload(self): + """Validate the modeler before upload.""" + self.base_sim.validate_pre_upload(source_required=False) + AbstractComponentModeler.update_forward_refs() diff --git a/tidy3d/plugins/smatrix/component_modelers/modal.py b/tidy3d/plugins/smatrix/component_modelers/modal.py index 591ad4c624..e127129644 100644 --- a/tidy3d/plugins/smatrix/component_modelers/modal.py +++ b/tidy3d/plugins/smatrix/component_modelers/modal.py @@ -59,6 +59,11 @@ class ModalComponentModeler(AbstractComponentModeler): "by ``element_mappings``, the simulation corresponding to this column is skipped automatically.", ) + @property + def base_sim(self): + """The base simulation.""" + return self.simulation + @cached_property def sim_dict(self) -> SimulationMap: """Generates all :class:`.Simulation` objects for the S-matrix calculation. diff --git a/tidy3d/plugins/smatrix/component_modelers/terminal.py b/tidy3d/plugins/smatrix/component_modelers/terminal.py index ccc2de5f22..af8462c89a 100644 --- a/tidy3d/plugins/smatrix/component_modelers/terminal.py +++ b/tidy3d/plugins/smatrix/component_modelers/terminal.py @@ -223,7 +223,7 @@ def _warn_refactor_2_10(cls, values): @property def _sim_with_sources(self) -> Simulation: - """Instance of :class:`.Simulation` with all sources and absorbers added for each port, for troubleshooting.""" + """Instance of :class:`.Simulation` with all sources and absorbers added for each port, for plotting.""" sources = [port.to_source(self._source_time) for port in self.ports] absorbers = [ @@ -231,7 +231,9 @@ def _sim_with_sources(self) -> Simulation: for port in self.ports if isinstance(port, WavePort) and port.absorber ] - return self.simulation.updated_copy(sources=sources, internal_absorbers=absorbers) + return self.simulation.updated_copy( + sources=sources, internal_absorbers=absorbers, validate=False + ) @equal_aspect @add_ax_if_none @@ -382,6 +384,10 @@ def matrix_indices_run_sim(self) -> tuple[NetworkIndex, ...]: def sim_dict(self) -> SimulationMap: """Generate all the :class:`.Simulation` objects for the port parameter calculation.""" + # Check base simulation for grid size at ports + TerminalComponentModeler._check_grid_size_at_ports(self.base_sim, self._lumped_ports) + TerminalComponentModeler._check_grid_size_at_wave_ports(self.base_sim, self._wave_ports) + sim_dict = {} # Now, create simulations with wave port sources and mode solver monitors for computing port modes for network_index in self.matrix_indices_run_sim: @@ -389,11 +395,6 @@ def sim_dict(self) -> SimulationMap: # update simulation sim_dict[task_name] = sim_with_src - # Check final simulations for grid size at ports - for _, sim in sim_dict.items(): - TerminalComponentModeler._check_grid_size_at_ports(sim, self._lumped_ports) - TerminalComponentModeler._check_grid_size_at_wave_ports(sim, self._wave_ports) - return SimulationMap(keys=tuple(sim_dict.keys()), values=tuple(sim_dict.values())) @cached_property @@ -414,7 +415,10 @@ def _base_sim_no_radiation_monitors(self) -> Simulation: # Make an initial simulation with new grid_spec to determine where LumpedPorts are snapped sim_wo_source = self.simulation.updated_copy( - grid_spec=grid_spec, lumped_elements=lumped_resistors + grid_spec=grid_spec, + lumped_elements=lumped_resistors, + validate=False, + deep=False, ) snap_centers = {} for port in self._lumped_ports: @@ -480,7 +484,11 @@ def _base_sim_no_radiation_monitors(self) -> Simulation: ) # update base simulation with updated set of shared components - sim_wo_source = sim_wo_source.copy(update=update_dict) + sim_wo_source = sim_wo_source.updated_copy( + **update_dict, + validate=False, + deep=False, + ) # extrude port structures sim_wo_source = self._extrude_port_structures(sim=sim_wo_source) @@ -527,7 +535,10 @@ def base_sim(self) -> Simulation: """The base simulation with all components added, including radiation monitors.""" base_sim_tmp = self._base_sim_no_radiation_monitors mnts_with_radiation = list(base_sim_tmp.monitors) + list(self._finalized_radiation_monitors) - return base_sim_tmp.updated_copy(monitors=mnts_with_radiation) + grid_spec = GridSpec.from_grid(base_sim_tmp.grid) + grid_spec.attrs["from_grid_spec"] = base_sim_tmp.grid_spec + # We skipped validations up to now, here we finally validate the base sim + return base_sim_tmp.updated_copy(monitors=mnts_with_radiation, grid_spec=grid_spec) def _generate_radiation_monitor( self, simulation: Simulation, auto_spec: DirectivityMonitorSpec @@ -712,7 +723,10 @@ def _add_source_to_sim(self, source_index: NetworkIndex) -> tuple[str, Simulatio ) task_name = self.get_task_name(port=port, mode_index=mode_index) - return (task_name, self.base_sim.updated_copy(sources=[port_source])) + return ( + task_name, + self.base_sim.updated_copy(sources=[port_source], validate=False, deep=False), + ) @cached_property def _source_time(self): @@ -958,6 +972,8 @@ def _extrude_port_structures(self, sim: Simulation) -> Simulation: sim = sim.updated_copy( grid_spec=GridSpec.from_grid(sim.grid), structures=[*sim.structures, *all_new_structures], + validate=False, + deep=False, ) return sim diff --git a/tidy3d/web/__init__.py b/tidy3d/web/__init__.py index 0cdc8942e5..dcdf44c9c3 100644 --- a/tidy3d/web/__init__.py +++ b/tidy3d/web/__init__.py @@ -30,7 +30,6 @@ load, load_simulation, monitor, - postprocess_start, real_cost, start, test, @@ -58,7 +57,6 @@ "load", "load_simulation", "monitor", - "postprocess_start", "real_cost", "run", "run_async", diff --git a/tidy3d/web/api/container.py b/tidy3d/web/api/container.py index 08092e3062..1745cc8f63 100644 --- a/tidy3d/web/api/container.py +++ b/tidy3d/web/api/container.py @@ -389,25 +389,7 @@ def status(self) -> str: """Return current status of :class:`Job`.""" if self.load_if_cached: return "success" - if web._is_modeler_batch(self.task_id): - detail = self.get_info() - status = detail.totalStatus.value - return status - else: - return self.get_info().status - - @property - def postprocess_status(self) -> Optional[str]: - """Return current postprocess status of :class:`Job` if it is a Component Modeler.""" - if web._is_modeler_batch(self.task_id): - detail = self.get_info() - return detail.postprocessStatus - else: - log.warning( - f"Task ID '{self.task_id}' is not a modeler batch job. " - "'postprocess_start' is only applicable to Component Modelers" - ) - return + return self.get_info().status def start(self, priority: Optional[int] = None) -> None: """Start running a :class:`Job`. @@ -454,40 +436,50 @@ def monitor(self) -> None: return web.monitor(self.task_id, verbose=self.verbose) - def download(self, path: PathLike = DEFAULT_DATA_PATH) -> None: + def download(self, path: Optional[PathLike] = None) -> None: """Download results of simulation. Parameters ---------- - path : PathLike = "./simulation_data.hdf5" + path : Optional[PathLike] = None Path to download data as ``.hdf5`` file (including filename). + If not provided, uses a task-type-specific default filename. Note ---- To load the data after download, use :meth:`Job.load`. """ + # For cached loads, we need a path - use default if not provided if self.load_if_cached: + if path is None: + path = DEFAULT_DATA_PATH self._materialize_from_stash(path) return - self._check_path_dir(path=path) + if path is not None: + self._check_path_dir(path=path) web.download(task_id=self.task_id, path=path, verbose=self.verbose) - def load(self, path: PathLike = DEFAULT_DATA_PATH) -> WorkflowDataType: + def load(self, path: Optional[PathLike] = None) -> WorkflowDataType: """Download job results and load them into a data object. Parameters ---------- - path : PathLike = "./simulation_data.hdf5" + path : Optional[PathLike] = None Path to download data as ``.hdf5`` file (including filename). + If not provided, uses a task-type-specific default filename. Returns ------- Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] Object containing simulation results. """ - self._check_path_dir(path=path) + # For cached loads, we need a path - use default if not provided if self.load_if_cached: + if path is None: + path = DEFAULT_DATA_PATH self._materialize_from_stash(path) + elif path is not None: + self._check_path_dir(path=path) data = web.load( task_id=None if self.load_if_cached else self.task_id, @@ -548,37 +540,6 @@ def estimate_cost(self, verbose: bool = True) -> float: return 0.0 return web.estimate_cost(self.task_id, verbose=verbose, solver_version=self.solver_version) - def postprocess_start(self, worker_group: Optional[str] = None, verbose: bool = True) -> None: - """ - If the job is a modeler batch, checks if the run is complete and starts - the postprocess phase. - - This function does not wait for postprocessing to finish and is only - applicable to Component Modeler batch jobs. - - Parameters - ---------- - worker_group : Optional[str] = None - The specific worker group to run the postprocessing task on. - verbose : bool = True - Whether to print info messages. This overrides the Job's 'verbose' setting for this call. - """ - # First, confirm that the task is a modeler batch job. - if not web._is_modeler_batch(self.task_id): - # If not, inform the user and exit. - # This warning is important and should not be suppressed. - log.warning( - f"Task ID '{self.task_id}' is not a modeler batch job. " - "'postprocess_start' is only applicable to Component Modelers" - ) - return - - # If it is a modeler batch, call the dedicated function to start postprocessing. - # The verbosity is a combination of the job's setting and the method's parameter. - web.postprocess_start( - batch_id=self.task_id, verbose=(self.verbose and verbose), worker_group=worker_group - ) - @staticmethod def _check_path_dir(path: PathLike) -> None: """Make sure parent directory of ``path`` exists and create it if not. @@ -1043,21 +1004,6 @@ def get_run_info(self) -> dict[TaskName, RunInfo]: run_info_dict[task_name] = run_info return run_info_dict - def postprocess_start(self, worker_group: Optional[str] = None, verbose: bool = True) -> None: - """ - Start the postprocess phase for all applicable jobs in the batch. - - This simply forwards to each Job's `postprocess_start(...)`. The Job decides - whether it's a Component Modeler task and whether it can/should start now. - This method does not wait for postprocessing to finish. - """ - if self.verbose and verbose: - console = get_logging_console() - console.log("Attempting to start postprocessing for jobs in the batch.") - - for job in self.jobs.values(): - job.postprocess_start(worker_group=worker_group, verbose=verbose) - def monitor( self, *, @@ -1069,7 +1015,6 @@ def monitor( """ Monitor progress of each running task. - - For Component Modeler jobs, automatically triggers postprocessing once run finishes. - Optionally downloads results as soon as a job reaches final success. - Rich progress bars in verbose mode; quiet polling otherwise. @@ -1095,16 +1040,8 @@ def monitor( self._check_path_dir(path_dir=path_dir) download_executor = ThreadPoolExecutor(max_workers=self.num_workers) - def _should_download(job: Job) -> bool: - status = job.status - if not web._is_modeler_batch(job.task_id): - return status == "success" - if status == "success": - return True - return status == "run_success" and getattr(job, "postprocess_status", None) == "success" - def schedule_download(job: Job) -> None: - if download_executor is None or not _should_download(job): + if download_executor is None or job.status not in COMPLETED_STATES: return task_id = job.task_id if task_id in downloads_started: @@ -1128,12 +1065,7 @@ def schedule_download(job: Job) -> None: def check_continue_condition(job: Job) -> bool: if job.load_if_cached: return False - status = job.status - if not web._is_modeler_batch(job.task_id): - return status not in END_STATES - if status == "run_success": - return job.postprocess_status not in END_STATES - return status not in END_STATES + return job.status not in END_STATES def pbar_description( task_name: str, status: str, max_name_length: int, status_width: int @@ -1157,9 +1089,6 @@ def pbar_description( max_task_name = max(len(task_name) for task_name in self.jobs.keys()) max_name_length = min(30, max(max_task_name, 15)) - # track which modeler jobs we've already kicked into postprocess - postprocess_started_tasks: set[str] = set() - try: console = None progress_columns = [] @@ -1195,17 +1124,6 @@ def pbar_description( for task_name, job in self.jobs.items(): status = job.status - # auto-start postprocess for modeler jobs when run finishes - if ( - web._is_modeler_batch(job.task_id) - and status == "run_success" - and job.task_id not in postprocess_started_tasks - ): - job.postprocess_start( - worker_group=postprocess_worker_group, verbose=True - ) - postprocess_started_tasks.add(job.task_id) - schedule_download(job) if self.verbose: diff --git a/tidy3d/web/api/states.py b/tidy3d/web/api/states.py index 9b02d161f6..55cf108e9c 100644 --- a/tidy3d/web/api/states.py +++ b/tidy3d/web/api/states.py @@ -5,15 +5,17 @@ } ERROR_STATES = { - "validate_fail", + "validate_error", "error", "errored", "diverge", "diverged", "blocked", - "run_failed", + "preprocess_error", + "run_error", "aborted", "deleted", + "postprocess_error", } PRE_VALIDATE_STATES = { @@ -30,7 +32,7 @@ "run_success", } -COMPLETED_STATES = {"visualize", "success", "completed", "processed"} +COMPLETED_STATES = {"visualize", "success", "completed", "processed", "postprocess_success"} END_STATES = ERROR_STATES | COMPLETED_STATES @@ -87,6 +89,7 @@ "visualize": round((11 / MAX_STEPS) * COMPLETED_PERCENT), # 85% "success": COMPLETED_PERCENT, # 100% "completed": COMPLETED_PERCENT, # 100% + "postprocess_success": COMPLETED_PERCENT, # 100% # --- Error States --- # All error states map to 0% "validate_fail": 0, diff --git a/tidy3d/web/api/tidy3d_stub.py b/tidy3d/web/api/tidy3d_stub.py index fc979a1e2d..dafc225d12 100644 --- a/tidy3d/web/api/tidy3d_stub.py +++ b/tidy3d/web/api/tidy3d_stub.py @@ -181,7 +181,7 @@ def validate_pre_upload(self, source_required: bool) -> None: """Perform some pre-checks on instances of component""" if isinstance(self.simulation, Simulation): self.simulation.validate_pre_upload(source_required) - elif isinstance(self.simulation, EMESimulation): + else: self.simulation.validate_pre_upload() def get_default_task_name(self) -> str: diff --git a/tidy3d/web/api/webapi.py b/tidy3d/web/api/webapi.py index a57f0ebbfe..e5040f0f3a 100644 --- a/tidy3d/web/api/webapi.py +++ b/tidy3d/web/api/webapi.py @@ -24,7 +24,6 @@ ALL_POST_VALIDATE_STATES, END_STATES, ERROR_STATES, - POST_VALIDATE_STATES, STATE_PROGRESS_PERCENTAGE, ) from tidy3d.web.cache import CacheEntry, _store_mode_solver_in_cache, resolve_local_cache @@ -39,11 +38,8 @@ SIMULATION_DATA_HDF5_GZ, TaskId, ) -from tidy3d.web.core.exceptions import WebNotFoundError -from tidy3d.web.core.http_util import get_version as _get_protocol_version -from tidy3d.web.core.http_util import http from tidy3d.web.core.task_core import BatchDetail, BatchTask, Folder, SimulationTask -from tidy3d.web.core.task_info import AsyncJobDetail, ChargeType, TaskInfo +from tidy3d.web.core.task_info import ChargeType, TaskInfo from tidy3d.web.core.types import PayType, TaskType from .connect_util import REFRESH_TIME, get_grid_points_str, get_time_steps_str, wait_for_connection @@ -72,6 +68,63 @@ "VOLUME_MESH": "VolumeMesher", } +# map task_type to default data filename +DEFAULT_DATA_FILENAME = { + "FDTD": "simulation_data.hdf5", + "MODE_SOLVER": "simulation_data.hdf5", + "MODE": "simulation_data.hdf5", + "EME": "simulation_data.hdf5", + "HEAT": "simulation_data.hdf5", + "HEAT_CHARGE": "simulation_data.hdf5", + "VOLUME_MESH": "simulation_data.hdf5", + "COMPONENT_MODELER": "cm_data.hdf5", + "TERMINAL_COMPONENT_MODELER": "cm_data.hdf5", + "RF": "cm_data.hdf5", +} + + +def _get_default_path( + task_id: str, provided_path: Optional[PathLike] +) -> tuple[Path, bool, Optional[str]]: + """Get the appropriate default path based on task type. + + If the user provided a path, returns it as-is. + If no path is provided (None), returns the task-type-specific default filename. + + Parameters + ---------- + task_id : str + Unique identifier of task on server. + provided_path : Optional[PathLike] + Path provided by the user, or None to use task-type default. + + Returns + ------- + tuple[Path, bool, Optional[str]] + A tuple containing: + - Path: The appropriate path to use for this task type + - bool: True if this is a modeler batch task + - Optional[str]: Task type string (None if user provided a path) + """ + # If user provided a path, respect it exactly + if provided_path is not None: + # Still need to determine if it's a modeler batch for later use + is_batch = _is_modeler_batch(task_id) + return Path(provided_path), is_batch, None + + # Determine task type for default filename + is_batch = _is_modeler_batch(task_id) + if is_batch: + task_type = "RF" + else: + task_info = get_info(task_id) + task_type = task_info.taskType + + # Get the task-type-specific default filename + default_filename = DEFAULT_DATA_FILENAME.get(task_type, "simulation_data.hdf5") + + return Path(default_filename), is_batch, task_type + def _get_url(task_id: str) -> str: """Get the URL for a task on our server.""" @@ -97,11 +150,7 @@ def _build_website_url(path: str) -> str: def _is_modeler_batch(resource_id: str) -> bool: """Detect whether the given id corresponds to a modeler batch resource.""" - return BatchTask.is_batch(resource_id, batch_type="RF_SWEEP") - - -def _batch_detail(resource_id: str) -> BatchDetail: - return BatchTask(resource_id).detail(batch_type="RF_SWEEP") + return BatchTask.is_batch(resource_id) def _batch_detail_error(resource_id: str) -> Optional[WebError]: @@ -115,26 +164,22 @@ def _batch_detail_error(resource_id: str) -> Optional[WebError]: Args: resource_id (str): The identifier of the batch resource that failed. - Returns: - An instance of `WebError` if the batch failed, otherwise `None`. + Raises: + An instance of ``WebError`` if the batch failed. """ + + # TODO: test properly try: - batch_detail = BatchTask(batch_id=resource_id).detail(batch_type="RF_SWEEP") - status = batch_detail.totalStatus.value + batch_detail = BatchTask(batch_id=resource_id).detail() + status = batch_detail.status.lower() except Exception as e: log.error(f"Could not retrieve batch details for '{resource_id}': {e}") - return WebError(f"Failed to retrieve status for batch '{resource_id}'.") + raise WebError(f"Failed to retrieve status for batch '{resource_id}'.") from e if status not in ERROR_STATES: - return None - - log.error(f"The ComponentModeler batch '{resource_id}' has failed with status: {status}") + return - if ( - status == "validate_fail" - and hasattr(batch_detail, "validateErrors") - and batch_detail.validateErrors - ): + if hasattr(batch_detail, "validateErrors") and batch_detail.validateErrors: error_details = [] for key, error_str in batch_detail.validateErrors.items(): try: @@ -153,7 +198,7 @@ def _batch_detail_error(resource_id: str) -> Optional[WebError]: "One or more subtasks failed validation. Please fix the component modeler configuration.\n" f"Details:\n{details_string}" ) - return WebError(full_error_msg) + raise WebError(full_error_msg) # Handle all other generic error states else: @@ -161,168 +206,7 @@ def _batch_detail_error(resource_id: str) -> Optional[WebError]: f"Batch '{resource_id}' failed with status '{status}'. Check server " "logs for details or contact customer support." ) - return WebError(error_msg) - - -def _upload_component_modeler_subtasks( - resource_id: str, verbose: bool = True, solver_version: Optional[str] = None -) -> Optional[WebError]: - """Kicks off and monitors the split and validation of component modeler tasks. - - This function orchestrates a two-phase process. First, it initiates a - server-side asynchronous job to split the components of a modeler batch. - It monitors this job's progress by polling the API and parsing the - response into an `AsyncJobDetail` model until the job completes or fails. - - If the split is successful, the function proceeds to the second phase: - triggering a batch validation via `batch.check()`. It then monitors this - validation process by polling for `BatchDetail` updates. The progress bar, - if verbose, reflects the status according to a predefined state mapping. - - Finally, it processes the terminal state of the validation. If a - 'validate_fail' status occurs, it parses detailed error messages for each - failed subtask and includes them in the raised exception. - - Args: - resource_id (str): The identifier for the batch resource to be processed. - verbose (bool): If True, displays progress bars and logs detailed - status messages to the console during the operation. - solver_version (str): Solver version in which to run validation. - - Raises: - RuntimeError: If the initial asynchronous split job fails. - WebError: If the subsequent batch validation fails, ends in an - unexpected state, or if a 'validate_fail' status is encountered. - """ - console = get_logging_console() if verbose else None - final_error = None - batch_type = "RF_SWEEP" - - split_path = "tidy3d/async-biz/component-modeler-split" - payload = { - "batchType": batch_type, - "batchId": resource_id, - "fileName": "modeler.hdf5.gz", - "protocolVersion": _get_protocol_version(), - } - - if verbose: - console.log("Starting modeler and subtasks validation...") - - initial_resp = http.post(split_path, payload) - split_job_detail = AsyncJobDetail(**initial_resp) - monitor_split_path = f"{split_path}?asyncId={split_job_detail.asyncId}" - - if verbose: - progress_bar = Progress( - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), - TimeElapsedColumn(), - console=console, - ) - - with progress_bar as progress: - description = "Upload Subtasks" - pbar = progress.add_task(description, completed=split_job_detail.progress, total=100) - while True: - split_job_raw_result = http.get(monitor_split_path) - split_job_detail = AsyncJobDetail(**split_job_raw_result) - - progress.update( - pbar, completed=split_job_detail.progress, description=f"[blue]{description}" - ) - - if split_job_detail.status in END_STATES: - progress.update( - pbar, - completed=split_job_detail.progress, - description=f"[green]{description}", - ) - break - time.sleep(RUN_REFRESH_TIME) - - if split_job_detail.status in ERROR_STATES: - msg = split_job_detail.result or "An unknown error occurred." - final_error = WebError( - f"Component modeler split job failed ({split_job_detail.status}): {msg}" - ) - - if not final_error: - description = "Validating" - pbar = progress.add_task( - completed=10, total=100, description=f"[blue]{description}" - ) - batch = BatchTask(resource_id) - batch.check(solver_version=solver_version, batch_type=batch_type) - - while True: - batch_detail = batch.detail(batch_type=batch_type) - status = batch_detail.totalStatus - progress_percent = STATE_PROGRESS_PERCENTAGE.get(status, 0) - progress.update( - pbar, completed=progress_percent, description=f"[blue]{description}" - ) - - if status in POST_VALIDATE_STATES: - progress.update(pbar, completed=100, description=f"[green]{description}") - task_mapping = json.loads(split_job_detail.result) - console.log( - f"Uploaded Subtasks: \n{_task_dict_to_url_bullet_list(task_mapping)}" - ) - progress.refresh() - break - elif status in ERROR_STATES: - progress.update(pbar, completed=0, description=f"[red]{description}") - progress.refresh() - break - time.sleep(RUN_REFRESH_TIME) - - else: - # Non-verbose mode: Poll for split job completion. - while True: - split_job_raw_result = http.get(monitor_split_path) - split_job_detail = AsyncJobDetail(**split_job_raw_result) - if split_job_detail.status in END_STATES: - break - time.sleep(RUN_REFRESH_TIME) - - # Check for split job failure. - if split_job_detail.status in ERROR_STATES: - msg = split_job_detail.result or "An unknown error occurred." - final_error = WebError( - f"Component modeler split job failed ({split_job_detail.status}): {msg}" - ) - - # If split succeeded, poll for validation completion. - if not final_error: - batch = BatchTask(resource_id) - batch.check(solver_version=solver_version, batch_type=batch_type) - while True: - batch_detail = batch.detail(batch_type=batch_type) - status = batch_detail.totalStatus - if status in POST_VALIDATE_STATES or status in END_STATES: - break - time.sleep(RUN_REFRESH_TIME) - - return _batch_detail_error(resource_id=resource_id) - - -def _task_dict_to_url_bullet_list(data_dict: dict) -> str: - """ - Converts a dictionary into a string formatted as a bullet point list. - - Args: - data_dict: The dictionary to convert. - - Returns: - A string with each key-url/value pair as a bullet point. - """ - # Use a list comprehension to format each key-value pair - # and then join them together with newline characters. - if data_dict is None: - raise WebError("Error in subtask dictionary data.") - return "\n".join([f"- {key}: '{value}'" for key, value in data_dict.items()]) + raise WebError(error_msg) def _copy_simulation_data_from_cache_entry(entry: CacheEntry, path: PathLike) -> bool: @@ -452,7 +336,7 @@ def run( simulation: WorkflowType, task_name: Optional[str] = None, folder_name: str = "default", - path: PathLike = "simulation_data.hdf5", + path: Optional[PathLike] = None, callback_url: Optional[str] = None, verbose: bool = True, progress_callback_upload: Optional[Callable[[float], None]] = None, @@ -478,8 +362,9 @@ def run( Name of task. If not provided, a default name will be generated. folder_name : str = "default" Name of folder to store task on web UI. - path : PathLike = "simulation_data.hdf5" + path : Optional[PathLike] = None Path to download results file (.hdf5), including filename. + If not provided, uses a task-type-specific default filename. callback_url : str = None Http PUT url to receive simulation finish event. The body content is a json file with fields ``{'id', 'status', 'name', 'workUnit', 'solverVersion'}``. @@ -504,7 +389,7 @@ def run( It affects only simulations from vGPU licenses and does not impact simulations using FlexCredits. lazy : bool = False Whether to load the actual data (``lazy=False``) or return a proxy that loads - the data when accessed (``lazy=True``). + the data when accessed (``lazy=True`). Returns ------- @@ -572,7 +457,6 @@ def run( ) start( task_id, - verbose=verbose, solver_version=solver_version, worker_group=worker_group, pay_type=pay_type, @@ -695,10 +579,8 @@ def upload( task_type = stub.get_type() # Component modeler compatibility: map to RF task type - port_name_list = None if task_type in ("COMPONENT_MODELER", "TERMINAL_COMPONENT_MODELER"): task_type = "RF" - port_name_list = tuple(simulation.sim_dict.keys()) task = SimulationTask.create( task_type, @@ -708,7 +590,6 @@ def upload( simulation_type, parent_tasks, "Gz", - port_name_list=port_name_list, ) if task_type == "RF": @@ -759,15 +640,10 @@ def upload( remote_sim_file=remote_sim_file, ) - if task_type == "RF": - _upload_component_modeler_subtasks(resource_id=resource_id, verbose=verbose) - estimate_cost(task_id=resource_id, solver_version=solver_version, verbose=verbose) task.validate_post_upload(parent_tasks=parent_tasks) - # log the url for the task in the web UI - log.debug(_build_website_url(f"folders/{task.folder_id}/tasks/{resource_id}")) return resource_id @@ -849,7 +725,7 @@ def get_info(task_id: TaskId, verbose: bool = True) -> TaskInfo | BatchDetail: """ if _is_modeler_batch(task_id): batch = BatchTask(task_id) - return batch.detail(batch_type="RF_SWEEP") + return batch.detail() else: task = SimulationTask.get(task_id, verbose) if not task: @@ -860,7 +736,6 @@ def get_info(task_id: TaskId, verbose: bool = True) -> TaskInfo | BatchDetail: @wait_for_connection def start( task_id: TaskId, - verbose: bool = True, solver_version: Optional[str] = None, worker_group: Optional[str] = None, pay_type: Union[PayType, str] = PayType.AUTO, @@ -889,30 +764,13 @@ def start( To monitor progress, can call :meth:`monitor` after starting simulation. """ - console = get_logging_console() if verbose else None - - # Component modeler batch path: hide split/check/submit - if _is_modeler_batch(task_id): - # split (modeler-specific) - batch = BatchTask(task_id) - detail = batch.wait_for_validate(batch_type="RF_SWEEP") - status = detail.totalStatus - status_str = status.value - if status_str in POST_VALIDATE_STATES: - pass - elif status_str not in POST_VALIDATE_STATES: - raise WebError(f"Batch task {task_id} is blocked: {status_str}") - # Submit batch to start runs after validation - batch.submit( - solver_version=solver_version, batch_type="RF_SWEEP", worker_group=worker_group - ) - if verbose: - console.log(f"Component Modeler '{task_id}' validated. Solving...") - return - if priority is not None and (priority < 1 or priority > 10): raise ValueError("Priority must be between '1' and '10' if specified.") - task = SimulationTask.get(task_id) + + if _is_modeler_batch(task_id): + task = BatchTask(task_id) + else: + task = SimulationTask.get(task_id) if not task: raise ValueError("Task not found.") task.submit( @@ -945,6 +803,16 @@ def get_run_info(task_id: TaskId) -> tuple[Optional[float], Optional[float]]: return task.get_running_info() +def _get_batch_detail_handle_error_status(task_id: TaskId) -> BatchDetail: + """Get batch detail and raise error if status is in ERROR_STATES.""" + batch = BatchTask(task_id) + detail = batch.detail() + status = detail.status.lower() + if status in ERROR_STATES: + raise _batch_detail_error(task_id) + return detail + + def get_status(task_id: TaskId) -> str: """Get the status of a task. Raises an error if status is "error". @@ -954,21 +822,7 @@ def get_status(task_id: TaskId) -> str: Unique identifier of task on server. Returned by :meth:`upload`. """ if _is_modeler_batch(task_id): - # split (modeler-specific) - batch = BatchTask(task_id) - detail = batch.detail(batch_type="RF_SWEEP") - status = detail.totalStatus - if status == "visualize": - return "success" - if status in ERROR_STATES: - try: - # TODO Try to obtain the error message - pass - except Exception: - # If the error message could not be obtained, raise a generic error message - error_msg = "Error message could not be obtained, please contact customer support." - - raise WebError(f"Error running task {task_id}! {error_msg}") + return _get_batch_detail_handle_error_status(task_id).status else: task_info = get_info(task_id) status = task_info.status @@ -1018,8 +872,7 @@ def monitor(task_id: TaskId, verbose: bool = True, worker_group: Optional[str] = # Batch/modeler monitoring path if _is_modeler_batch(task_id): - _monitor_modeler_batch(task_id, verbose=verbose, worker_group=worker_group) - return + return _monitor_modeler_batch(task_id, verbose=verbose) console = get_logging_console() if verbose else None @@ -1178,35 +1031,30 @@ def abort(task_id: TaskId) -> Optional[TaskInfo]: Object containing information about status, size, credits of task. """ console = get_logging_console() - try: - task = SimulationTask.get(task_id, verbose=False) - if task: - task.abort() - url = _get_url(task.task_id) - console.log( - f"Task is aborting. View task using web UI at [link={url}]'{url}'[/link] to check the result." - ) - return TaskInfo(**{"taskId": task.task_id, **task.dict()}) - except WebNotFoundError: - pass # Task not found, might be a batch task - is_batch = BatchTask.is_batch(task_id, batch_type="RF_SWEEP") - if is_batch: + if _is_modeler_batch(task_id): + task = BatchTask(task_id) url = _get_url_rf(task_id) + else: + task = SimulationTask.get(task_id, verbose=False) + url = _get_url(task_id) + + if task: + task.abort() console.log( - f"Batch task abortion is not yet supported, contact customer support." - f" View task using web UI at [link={url}]'{url}'[/link]." + f"Task is aborting. View task using web UI at [link={url}]'{url}'[/link] to check the result." ) - return - - console.log("Task ID cannot be found to be aborted.") - return + if _is_modeler_batch(task_id): + detail = task.detail() + return TaskInfo(**{"taskId": task_id, "taskType": "RF", **detail.dict()}) + else: + return TaskInfo(**{"taskId": task_id, **task.dict()}) @wait_for_connection def download( task_id: TaskId, - path: PathLike = "simulation_data.hdf5", + path: Optional[PathLike] = None, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, ) -> None: @@ -1216,57 +1064,32 @@ def download( ---------- task_id : str Unique identifier of task on server. Returned by :meth:`upload`. - path : PathLike = "simulation_data.hdf5" + path : Optional[PathLike] = None Download path to .hdf5 data file (including filename). + If not provided, uses a task-type-specific default filename. verbose : bool = True If ``True``, will print progressbars and status, otherwise, will run silently. progress_callback : Callable[[float], None] = None Optional callback function called when downloading file with ``bytes_in_chunk`` as argument. """ - path = Path(path) - - if _is_modeler_batch(task_id): - # Use a more descriptive default filename for component modeler downloads. - # If the caller left the default as 'simulation_data.hdf5', prefer 'cm_data.hdf5'. - if path.name == "simulation_data.hdf5": - path = path.with_name("cm_data.hdf5") + # Get the appropriate default path based on task type + path, is_batch, task_type = _get_default_path(task_id, path) - def _download_cm() -> bool: - try: - BatchTask(task_id).get_data_hdf5( - remote_data_file_gz=CM_DATA_HDF5_GZ, - to_file=path, - verbose=verbose, - progress_callback=progress_callback, - ) - return True - except Exception: - return False - - if not _download_cm(): - BatchTask(task_id).postprocess(batch_type="RF_SWEEP") - # wait for postprocess to finish - while True: - resp = BatchTask(task_id).detail(batch_type="RF_SWEEP") - total = resp.totalTask or 0 - post_succ = resp.postprocessSuccess or 0 - status = resp.totalStatus - status_str = status.value - if status_str in ERROR_STATES: - raise WebError( - f"Batch task {task_id} failed during postprocess: {status_str}" - ) from None - if total > 0 and post_succ >= total: - break - time.sleep(REFRESH_TIME) - if not _download_cm(): - raise WebError("Failed to download 'cm_data' after postprocess completion.") + if is_batch: + BatchTask(task_id).get_data_hdf5( + remote_data_file_gz=CM_DATA_HDF5_GZ, + to_file=path, + verbose=verbose, + progress_callback=progress_callback, + ) return # Regular single-task download - task_info = get_info(task_id) - task_type = task_info.taskType + # Only call get_info if we don't already have task_type + if task_type is None: + task_info = get_info(task_id) + task_type = task_info.taskType remote_data_file = SIMULATION_DATA_HDF5_GZ if task_type == "MODE_SOLVER": @@ -1368,7 +1191,7 @@ def download_log( @wait_for_connection def load( task_id: Optional[TaskId], - path: PathLike = "simulation_data.hdf5", + path: Optional[PathLike] = None, replace_existing: bool = True, verbose: bool = True, progress_callback: Optional[Callable[[float], None]] = None, @@ -1394,8 +1217,10 @@ def load( ---------- task_id : Optional[str] = None Unique identifier of task on server. Returned by :meth:`upload`. If None, file is assumed to exist already from cache. - path : PathLike + path : Optional[PathLike] = None Download path to .hdf5 data file (including filename). + If not provided and task_id is given, uses a task-type-specific default filename. + If not provided and task_id is None, defaults to "simulation_data.hdf5". replace_existing : bool = True Downloads the data even if path exists (overwriting the existing). verbose : bool = True @@ -1411,14 +1236,14 @@ def load( Union[:class:`.SimulationData`, :class:`.HeatSimulationData`, :class:`.EMESimulationData`] Object containing simulation data. """ - path = Path(path) - # For component modeler batches, default to a clearer filename if the default was used. - if ( - task_id - and _is_modeler_batch(task_id) - and path.name in {"simulation_data.hdf5", "simulation_data.hdf5.gz"} - ): - path = path.with_name(path.name.replace("simulation", "cm")) + # Get the appropriate default path based on task type + if task_id is not None: + path, is_batch, task_type = _get_default_path(task_id, path) + else: + # When no task_id, use provided path or fall back to generic default + path = Path(path) if path is not None else Path("simulation_data.hdf5") + is_batch = False + task_type = None if task_id is None: if not path.exists(): @@ -1428,8 +1253,8 @@ def load( if verbose and task_id is not None: console = get_logging_console() - if _is_modeler_batch(task_id): - console.log(f"loading component modeler data from {path}") + if is_batch: + console.log(f"Loading component modeler data from {path}") else: console.log(f"Loading simulation from {path}") @@ -1437,8 +1262,12 @@ def load( simulation_cache = resolve_local_cache() if simulation_cache is not None and task_id is not None: - info = get_info(task_id, verbose=False) - workflow_type = getattr(info, "taskType", None) + # Only call get_info if we don't already have task_type + if task_type is None: + info = get_info(task_id, verbose=False) + workflow_type = getattr(info, "taskType", None) + else: + workflow_type = task_type if ( workflow_type != TaskType.MODE_SOLVER.name ): # we cannot get the simulation from data or web for mode solver @@ -1461,74 +1290,48 @@ def load( return stub_data +def _status_to_stage(status: str) -> tuple[str, int]: + """Map task status to monotonic stage for progress bars.""" + s = (status or "").lower() + # Map a broader set of states to monotonic stages for progress bars + if s in ("draft", "created"): + return ("draft", 0) + if s in ("queue", "queued"): + return ("queued", 1) + if s in ("validating",): + return ("validating", 2) + if s in ("validate_success", "validate_warn", "preprocess", "preprocessing"): + return ("preprocess", 3) + if s in ("running", "preprocess_success"): + return ("running", 4) + if s in ("run_success", "postprocess"): + return ("postprocess", 5) + if s in ("success", "postprocess_success"): + return ("success", 6) + # Unknown states map to earliest stage to avoid showing 100% prematurely + return (s or "unknown", 0) + + def _monitor_modeler_batch( batch_id: str, verbose: bool = True, max_detail_tasks: int = 20, - worker_group: Optional[str] = None, ) -> None: """Monitor modeler batch progress with aggregate and per-task views.""" console = get_logging_console() if verbose else None - - def _status_to_stage(status: str) -> tuple[str, int]: - s = (status or "").lower() - # Map a broader set of states to monotonic stages for progress bars - if s in ("draft", "created"): - return ("draft", 0) - if s in ("queue", "queued"): - return ("queued", 1) - if s in ("preprocess",): - return ("preprocess", 1) - if s in ("validating",): - return ("validating", 2) - if s in ("validate_success", "validate_warn"): - return ("validate", 3) - if s in ("running",): - return ("running", 4) - if s in ("postprocess",): - return ("postprocess", 5) - if s in ("run_success", "success"): - return ("success", 6) - # Unknown states map to earliest stage to avoid showing 100% prematurely - return (s or "unknown", 0) - - detail = _batch_detail(batch_id) + detail = _get_batch_detail_handle_error_status(batch_id) name = detail.name or "modeler_batch" group_id = detail.groupId - - header = f"Subtasks status - {name}" - if group_id: - header += f"\nGroup ID: '{group_id}'" - if console is not None: - console.log(header) + status = detail.status.lower() # Non-verbose path: poll without progress bars then return if not verbose: # Run phase - while True: - d = _batch_detail(batch_id) - s = d.totalStatus.value - total = d.totalTask or 0 - r = d.runSuccess or 0 - if s in ERROR_STATES: - raise WebError(f"Batch {batch_id} terminated: {s}") - # Updated break condition for robustness - if s in ("run_success", "success") or (total and r >= total): - break + while _status_to_stage(status)[0] not in END_STATES: time.sleep(REFRESH_TIME) + detail = _get_batch_detail_handle_error_status(batch_id) + status = detail.status.lower() - postprocess_start(batch_id, verbose=False, worker_group=worker_group) - - while True: - d = _batch_detail(batch_id) - postprocess_status = d.postprocessStatus - if postprocess_status == "success": - break - elif postprocess_status in ERROR_STATES: - raise WebError( - f"Batch {batch_id} terminated. Please contact customer support and provide this Component Modeler batch ID: '{batch_id}'" - ) - time.sleep(REFRESH_TIME) return progress_columns = ( @@ -1537,16 +1340,25 @@ def _status_to_stage(status: str) -> tuple[str, int]: TaskProgressColumn(), TimeElapsedColumn(), ) + # Make the header + header = f"Subtasks status - {name}" + if group_id: + header += f"\nGroup ID: '{group_id}'" + console.log(header) with Progress(*progress_columns, console=console, transient=False) as progress: # Phase: Run (aggregate + per-task) p_run = progress.add_task("Run Total", total=1.0) task_bars: dict[str, int] = {} + prev_status = status + console.log(f"Batch status = {status}") - while True: - detail = _batch_detail(batch_id) - status = detail.totalStatus.value - total = detail.totalTask or 0 + # Note: get_status errors if an erroring status occurred + while _status_to_stage(status)[0] not in END_STATES: + total = len(detail.tasks) r = detail.runSuccess or 0 + if status != prev_status: + prev_status = status + console.log(f"Batch status = {_status_to_stage(status)[0]}") # Create per-task bars as soon as tasks appear if total and total <= max_detail_tasks and detail.tasks: @@ -1593,36 +1405,10 @@ def _status_to_stage(status: str) -> tuple[str, int]: refresh=False, ) - # Updated break condition for robustness - if status in ("run_success", "success") or (total and r >= total): - break - if status in ERROR_STATES: - raise WebError(f"Batch {batch_id} terminated: {status}") - progress.refresh() - time.sleep(REFRESH_TIME) - - postprocess_start(batch_id, verbose=True, worker_group=worker_group) - - p_post = progress.add_task("Postprocess", total=1.0) - while True: - detail = _batch_detail(batch_id) - postprocess_status = detail.postprocessStatus - if postprocess_status == "success": - progress.update(p_post, completed=1.0) - progress.refresh() - break - elif postprocess_status == "queued": - progress.update(p_post, completed=0.22) - elif postprocess_status == "preprocess": - progress.update(p_post, completed=0.33) - elif postprocess_status == "running": - progress.update(p_post, completed=0.55) - elif postprocess_status in ERROR_STATES: - raise WebError( - f"Batch {batch_id} terminated. Please contact customer support and provide this Component Modeler batch ID: '{batch_id}'" - ) progress.refresh() time.sleep(REFRESH_TIME) + detail = _get_batch_detail_handle_error_status(batch_id) + status = detail.status.lower() if console is not None: console.log("Modeler has finished running successfully.") @@ -1780,11 +1566,17 @@ def estimate_cost( console = get_logging_console() if verbose else None if _is_modeler_batch(task_id): - d = _batch_detail(task_id) - status = d.totalStatus.value + batch = BatchTask(task_id) + _ = batch.check(solver_version=solver_version) + detail = batch.detail() + status = detail.status.lower() + while status not in ALL_POST_VALIDATE_STATES: + time.sleep(REFRESH_TIME) + detail = batch.detail() + status = detail.status.lower() if status in ALL_POST_VALIDATE_STATES: - est_flex_unit = _batch_detail(task_id).estFlexUnit + est_flex_unit = detail.estFlexUnit if verbose: console.log( f"Maximum FlexCredit cost: {est_flex_unit:1.3f}. Minimum cost depends on " @@ -1793,8 +1585,8 @@ def estimate_cost( ) return est_flex_unit - elif status in ERROR_STATES: - return _batch_detail_error(resource_id=task_id) + if status in ERROR_STATES: + raise _batch_detail_error(resource_id=task_id) raise WebError("Could not get estimated cost!") @@ -1891,42 +1683,24 @@ def real_cost(task_id: str, verbose: bool = True) -> float | None: ) console = get_logging_console() if verbose else None - if _is_modeler_batch(task_id): - status = _batch_detail(task_id).totalStatus.value - flex_unit = _batch_detail(task_id).realFlexUnit or None - if (status not in ["success", "run_success"]) or (flex_unit is None): - log.warning( - f"Billed FlexCredit for task '{task_id}' is not available. If the task has been " - "successfully run, it should be available shortly. If this issue persists, contact customer support." - ) - else: - if verbose: + task_info = get_info(task_id) + flex_unit = task_info.realFlexUnit + ori_flex_unit = getattr(task_info, "oriRealFlexUnit", flex_unit) + if not flex_unit: + log.warning( + f"Billed FlexCredit for task '{task_id}' is not available. If the task has been " + "successfully run, it should be available shortly." + ) + else: + if verbose: + console.log(f"Billed flex credit cost: {flex_unit:1.3f}.") + if flex_unit != ori_flex_unit and task_info.taskType == "FDTD": console.log( - f"Billed FlexCredit cost: {flex_unit:1.3f}. Minimum cost depends on " - "task execution details. Use 'web.real_cost(task_id)' to get the billed FlexCredit " - "cost after a simulation run." + "Note: the task cost pro-rated due to early shutoff was below the minimum " + "threshold, due to fast shutoff. Decreasing the simulation 'run_time' should " + "decrease the estimated, and correspondingly the billed cost of such tasks." ) - - return flex_unit - else: - task_info = get_info(task_id) - flex_unit = task_info.realFlexUnit - ori_flex_unit = task_info.oriRealFlexUnit - if not flex_unit: - log.warning( - f"Billed FlexCredit for task '{task_id}' is not available. If the task has been " - "successfully run, it should be available shortly." - ) - else: - if verbose: - console.log(f"Billed flex credit cost: {flex_unit:1.3f}.") - if flex_unit != ori_flex_unit and task_info.taskType == "FDTD": - console.log( - "Note: the task cost pro-rated due to early shutoff was below the minimum " - "threshold, due to fast shutoff. Decreasing the simulation 'run_time' should " - "decrease the estimated, and correspondingly the billed cost of such tasks." - ) - return flex_unit + return flex_unit @wait_for_connection @@ -1986,49 +1760,6 @@ def account(verbose: bool = True) -> Account: return account_info -@wait_for_connection -def postprocess_start( - batch_id: str, - verbose: bool = True, - worker_group: Optional[str] = None, -) -> None: - """ - Checks if a batch run is complete and starts the postprocess phase. - - This function does not wait for postprocessing to finish. - """ - console = get_logging_console() if verbose else None - if _is_modeler_batch(batch_id): - # Perform a single check on the run phase status - detail = _batch_detail(batch_id) - status = detail.totalStatus.value - total_tasks = detail.totalTask or 0 - successful_runs = detail.runSuccess or 0 - - if status in ERROR_STATES: - raise WebError(f"Batch '{batch_id}' terminated with error status: {status}") - - # Check if the run phase is complete before proceeding - is_run_complete = status in ("run_success", "success") or ( - total_tasks > 0 and successful_runs >= total_tasks - ) - - if not is_run_complete: - if console: - console.log( - f"Batch '{batch_id}' run phase is not yet complete (Status: {status}). " - f"Cannot start postprocessing." - ) - return # Exit if the run is not done - BatchTask(batch_id).postprocess(batch_type="RF_SWEEP", worker_group=worker_group) - return - else: - raise WebError( - f"Batch ID '{batch_id}' is not a component modeler batch job. " - "'postprocess_start' is only applicable to those classes." - ) - - @wait_for_connection def test() -> None: """Confirm whether Tidy3D authentication is configured. diff --git a/tidy3d/web/core/http_util.py b/tidy3d/web/core/http_util.py index c7f7983bb5..69a8a48ae9 100644 --- a/tidy3d/web/core/http_util.py +++ b/tidy3d/web/core/http_util.py @@ -43,6 +43,7 @@ class ResponseCodes(Enum): def get_version() -> str: """Get the version for the current environment.""" return core_config.get_version() + # return "2.10.0rc2.1" def get_user_agent() -> str: diff --git a/tidy3d/web/core/task_core.py b/tidy3d/web/core/task_core.py index e3d7db1a97..4af7132d84 100644 --- a/tidy3d/web/core/task_core.py +++ b/tidy3d/web/core/task_core.py @@ -5,7 +5,6 @@ import os import pathlib import tempfile -import time from datetime import datetime from os import PathLike from typing import Callable, Optional, Union @@ -17,7 +16,6 @@ import tidy3d as td from tidy3d.config import config from tidy3d.exceptions import ValidationError -from tidy3d.web.common import REFRESH_TIME from . import http_util from .cache import FOLDER_CACHE @@ -211,7 +209,6 @@ def create( simulation_type: str = "tidy3d", parent_tasks: Optional[list[str]] = None, file_type: str = "Gz", - port_name_list: Optional[list[str]] = None, projects_endpoint: str = "tidy3d/projects", ) -> SimulationTask: """Create a new task on the server. @@ -246,20 +243,24 @@ def create( simulation_type = "tidy3d" folder = Folder.get(folder_name, create=True) - payload = { - "taskName": task_name, - "taskType": task_type, - "callbackUrl": callback_url, - "simulationType": simulation_type, - "parentTasks": parent_tasks, - "fileType": file_type, - } - # Component modeler: include port names if provided - if port_name_list: - # Align with backend contract: expect 'portNames' (not 'portNameList') - payload["portNames"] = port_name_list - - resp = http.post(f"{projects_endpoint}/{folder.folder_id}/tasks", payload) + + if task_type == "RF": + payload = { + "groupName": task_name, + "folderId": folder.folder_id, + "fileType": file_type, + } + resp = http.post("rf/task", payload) + else: + payload = { + "taskName": task_name, + "taskType": task_type, + "callbackUrl": callback_url, + "simulationType": simulation_type, + "parentTasks": parent_tasks, + "fileType": file_type, + } + resp = http.post(f"{projects_endpoint}/{folder.folder_id}/tasks", payload) # RF group creation may return group-level info without 'taskId'. # Use 'groupId' (or 'batchId' as fallback) as the resource id for subsequent uploads. if "taskId" not in resp and task_type == "RF": @@ -694,7 +695,7 @@ def get_error_json(self, to_file: PathLike, verbose: bool = True) -> pathlib.Pat ) def abort(self) -> requests.Response: - """Aborting current task from server.""" + """Abort the current task on the server.""" if not self.task_id: raise ValueError("Task id not found.") return http.put( @@ -737,17 +738,13 @@ class BatchTask: This class acts as a wrapper around the API endpoints for a specific batch, allowing users to check, submit, monitor, and download data from it. - - Note: - The 'batch_type' (e.g., "RF_SWEEP") must be provided by the caller to - most methods, as it dictates which backend service handles the request. """ def __init__(self, batch_id: str) -> None: self.batch_id = batch_id @staticmethod - def is_batch(resource_id: str, batch_type: str) -> bool: + def is_batch(resource_id: str) -> bool: """Checks if a given resource ID corresponds to a valid batch task. This is a utility function to verify a batch task's existence before @@ -757,8 +754,6 @@ def is_batch(resource_id: str, batch_type: str) -> bool: ---------- resource_id : str The unique identifier for the resource. - batch_type : str - The type of the batch to check (e.g., "RF_SWEEP"). Returns ------- @@ -769,8 +764,7 @@ def is_batch(resource_id: str, batch_type: str) -> bool: # TODO PROPERLY FIXME # Disable non critical logs due to check for resourceId, until we have a dedicated API for this resp = http.get( - f"tidy3d/tasks/{resource_id}/batch-detail", - params={"batchType": batch_type}, + f"rf/task/{resource_id}/statistics", suppress_404=True, ) status = bool(resp and isinstance(resp, dict) and "status" in resp) @@ -778,22 +772,16 @@ def is_batch(resource_id: str, batch_type: str) -> bool: except Exception: return False - def detail(self, batch_type: str) -> BatchDetail: + def detail(self) -> BatchDetail: """Fetches the detailed information and status of the batch. - Parameters - ---------- - batch_type : str - The type of the batch (e.g., "RF_SWEEP"). - Returns ------- BatchDetail An object containing the batch's latest data. """ resp = http.get( - f"tidy3d/tasks/{self.batch_id}/batch-detail", - params={"batchType": batch_type}, + f"rf/task/{self.batch_id}/statistics", ) # Some backends may return null for collection fields; coerce to sensible defaults if isinstance(resp, dict): @@ -805,7 +793,6 @@ def check( self, solver_version: Optional[str] = None, protocol_version: Optional[str] = None, - batch_type: str = "", ) -> requests.Response: """Submits a request to validate the batch configuration on the server. @@ -815,8 +802,6 @@ def check( The version of the solver to use for validation. protocol_version : Optional[str], default=None The data protocol version. Defaults to the current version. - batch_type : str, default="" - The type of the batch (e.g., "RF_SWEEP"). Returns ------- @@ -826,9 +811,8 @@ def check( if protocol_version is None: protocol_version = _get_protocol_version() return http.post( - f"tidy3d/projects/{self.batch_id}/batch-check", + f"rf/task/{self.batch_id}/check", { - "batchType": batch_type, "solverVersion": solver_version, "protocolVersion": protocol_version, }, @@ -839,7 +823,8 @@ def submit( solver_version: Optional[str] = None, protocol_version: Optional[str] = None, worker_group: Optional[str] = None, - batch_type: str = "", + pay_type: Union[PayType, str] = PayType.AUTO, + priority: Optional[int] = None, ) -> requests.Response: """Submits the batch for execution on the server. @@ -851,136 +836,34 @@ def submit( The data protocol version. Defaults to the current version. worker_group : Optional[str], default=None Optional identifier for a specific worker group to run on. - batch_type : str, default="" - The type of the batch (e.g., "RF_SWEEP"). Returns ------- Any The server's response to the submit request. """ - if protocol_version is None: - protocol_version = _get_protocol_version() - return http.post( - f"tidy3d/projects/{self.batch_id}/batch-submit", - { - "batchType": batch_type, - "solverVersion": solver_version, - "protocolVersion": protocol_version, - "workerGroup": worker_group, - }, - ) - - def postprocess( - self, - solver_version: Optional[str] = None, - protocol_version: Optional[str] = None, - worker_group: Optional[str] = None, - batch_type: str = "", - ) -> requests.Response: - """Initiates post-processing for a completed batch run. - Parameters - ---------- - solver_version : Optional[str], default=None - The version of the solver to use for post-processing. - protocol_version : Optional[str], default=None - The data protocol version. Defaults to the current version. - worker_group : Optional[str], default=None - Optional identifier for a specific worker group to run on. - batch_type : str, default="" - The type of the batch (e.g., "RF_SWEEP"). + # TODO: add support for pay_type and priority arguments + if pay_type != PayType.AUTO: + raise NotImplementedError( + "The 'pay_type' argument is not yet supported and will be ignored." + ) + if priority is not None: + raise NotImplementedError( + "The 'priority' argument is not yet supported and will be ignored." + ) - Returns - ------- - Any - The server's response to the post-process request. - """ if protocol_version is None: protocol_version = _get_protocol_version() return http.post( - f"tidy3d/projects/{self.batch_id}/postprocess", + f"rf/task/{self.batch_id}/submit", { - "batchType": batch_type, "solverVersion": solver_version, "protocolVersion": protocol_version, "workerGroup": worker_group, }, ) - def wait_for_validate( - self, timeout: Optional[float] = None, batch_type: str = "" - ) -> BatchDetail: - """Waits for the batch to complete the validation stage by polling its status. - - Parameters - ---------- - timeout : Optional[float], default=None - Maximum time in seconds to wait. If ``None``, waits indefinitely. - batch_type : str, default="" - The type of the batch (e.g., "RF_SWEEP"). - - Returns - ------- - BatchDetail - The final object after validation completes or a timeout occurs. - - Notes - ----- - This method blocks until the batch status is 'validate_success', - 'validate_warn', 'validate_fail', or another terminal state like 'blocked' - or 'aborted', or until the timeout is reached. - """ - start = datetime.now().timestamp() - while True: - d = self.detail(batch_type=batch_type) - status = d.totalStatus - if status in ("validate_success", "validate_warn", "validate_fail"): - return d - if status in ("blocked", "aborting", "aborted"): - return d - if timeout is not None and (datetime.now().timestamp() - start) > timeout: - return d - time.sleep(REFRESH_TIME) - - def wait_for_run(self, timeout: Optional[float] = None, batch_type: str = "") -> BatchDetail: - """Waits for the batch to complete the execution stage by polling its status. - - Parameters - ---------- - timeout : Optional[float], default=None - Maximum time in seconds to wait. If ``None``, waits indefinitely. - batch_type : str, default="" - The type of the batch (e.g., "RF_SWEEP"). - - Returns - ------- - BatchDetail - The final object after the run completes or a timeout occurs. - - Notes - ----- - This method blocks until the batch status reaches a terminal run state like - 'run_success', 'run_failed', 'diverged', 'blocked', or 'aborted', - or until the timeout is reached. - """ - start = datetime.now().timestamp() - while True: - d = self.detail(batch_type=batch_type) - status = d.totalStatus - if status in ( - "run_success", - "run_failed", - "diverged", - "blocked", - "aborting", - "aborted", - ): - return d - if timeout is not None and (datetime.now().timestamp() - start) > timeout: - return d - time.sleep(REFRESH_TIME) - def get_data_hdf5( self, remote_data_file_gz: str, @@ -1047,3 +930,9 @@ def get_data_hdf5( ) from e return file + + def abort(self) -> requests.Response: + """Abort the current task on the server.""" + if not self.batch_id: + raise ValueError("Batch id not found.") + return http.put(f"rf/task/{self.batch_id}/abort", {}) diff --git a/tidy3d/web/core/task_info.py b/tidy3d/web/core/task_info.py index 64cf00ea0f..f4582ff40d 100644 --- a/tidy3d/web/core/task_info.py +++ b/tidy3d/web/core/task_info.py @@ -10,31 +10,6 @@ import pydantic.v1 as pydantic -class TaskStatus(Enum): - """The statuses that the task can be in.""" - - INIT = "initialized" - """The task has been initialized.""" - - QUEUE = "queued" - """The task is in the queue.""" - - PRE = "preprocessing" - """The task is in the preprocessing stage.""" - - RUN = "running" - """The task is running.""" - - POST = "postprocessing" - """The task is in the postprocessing stage.""" - - SUCCESS = "success" - """The task has completed successfully.""" - - ERROR = "error" - """The task has completed with an error.""" - - class TaskBase(pydantic.BaseModel, ABC): """Base configuration for all task objects.""" @@ -172,41 +147,6 @@ def display(self) -> None: # ---------------------- Batch (Modeler) detail schema ---------------------- # -class BatchStatus(str, Enum): - """Enumerates the possible statuses for a batch of tasks.""" - - draft = "draft" - """The batch is being configured and has not been submitted.""" - preprocess = "preprocess" - """The batch is undergoing preprocessing.""" - validating = "validating" - """The tasks within the batch are being validated.""" - validate_success = "validate_success" - """All tasks in the batch passed validation.""" - validate_warn = "validate_warn" - """Validation passed, but with warnings.""" - validate_fail = "validate_fail" - """Validation failed for one or more tasks.""" - blocked = "blocked" - """The batch is blocked and cannot run.""" - running = "running" - """The batch is currently executing.""" - aborting = "aborting" - """The batch is in the process of being aborted.""" - run_success = "run_success" - """The batch completed successfully.""" - postprocess = "postprocess" - """The batch is undergoing postprocessing.""" - run_failed = "run_failed" - """The batch execution failed.""" - diverged = "diverged" - """The simulation in the batch diverged.""" - aborted = "aborted" - """The batch was successfully aborted.""" - error = "error" - """An error occurred during the solver run.""" - - class BatchTaskBlockInfo(TaskBlockInfo): """ Extends `TaskBlockInfo` with specific details for batch task blocking. @@ -282,7 +222,6 @@ class BatchDetail(TaskBase): groupId: Identifier for the group the batch belongs to. name: The user-defined name of the batch. status: The current status of the batch. - totalStatus: The overall status, consolidating individual task statuses. totalTask: The total number of tasks in the batch. preprocessSuccess: The count of tasks that completed preprocessing. postprocessStatus: The status of the batch's postprocessing stage. @@ -295,6 +234,7 @@ class BatchDetail(TaskBase): totalCheckMillis: Total time in milliseconds spent on checks. message: A general message providing information about the batch status. tasks: A list of `BatchMember` objects, one for each task in the batch. + taskType: The type of tasks contained in the batch. """ refId: str = None @@ -302,7 +242,6 @@ class BatchDetail(TaskBase): groupId: str = None name: str = None status: str = None - totalStatus: BatchStatus = None totalTask: int = 0 preprocessSuccess: int = 0 postprocessStatus: str = None @@ -317,6 +256,7 @@ class BatchDetail(TaskBase): message: str = None tasks: list[BatchMember] = [] validateErrors: dict = None + taskType: str = "RF" class AsyncJobDetail(TaskBase):