esm_runscripts package

Top-level package for ESM Runscripts.

Submodules

esm_runscripts.cli module

A small wrapper that combines the shell interface and the Python interface

esm_runscripts.cli.main()

esm_runscripts.cli.parse_shargs()

The arg parser for interactive use

esm_runscripts.compute module

Class to hold compute jobs and recipe steps

class esm_runscripts.compute.compute(config)

Bases: esm_runscripts.jobclass.jobclass

static add_batch_hostfile(config)

copy_files_to_thisrun()

copy_files_to_work()

static copy_tools_to_thisrun(config)

static create_new_files(config)

static initialize_experiment_logfile(config)

Initializes the log file for the entire experiment.

Creates a file ${BASE_DIR}/${EXPID}/log/${EXPID}_${setup_name}.log to keep track of start/stop times, job id numbers, and so on. Use the function write_to_log to put information in this file afterwards.

The user can specify experiment_log_file under the general section of the configuration to override the default name. Timestamps for each message are given by the section experiment_log_file_dateformat, or defaults to Tue Mar 17 09:36:38 2020, i.e. "%c". Please use stftime compatable formats, as described here: https://strftime.org

Parameters:dict – The experiment configuration Returns:As per convention for the plug-in system; this gives back the entire config. Return type:dict

Attention

Calling this has some filesystem side effects. If the run number in the general configuration is set to 1, and a file exists for general.exp_log_file; this file is removed; and re-initialized.

static modify_files(config)

static modify_namelists(config)

static prepare_coupler_files(config)

esm_runscripts.database module

class esm_runscripts.database.experiment(**kwargs)

Bases: sqlalchemy.ext.declarative.api.Base

archive_folder

cpuh

exp_folder

expid

gb

id

static nicer_output(run)

outcome

run_timestamp

runtime

setup_name

timestamp

static topline()

esm_runscripts.database_actions module

esm_runscripts.database_actions.database_basic_entry(config)

esm_runscripts.database_actions.database_entry(config)

esm_runscripts.database_actions.database_entry_check(config)

esm_runscripts.database_actions.database_entry_crashed(config)

esm_runscripts.database_actions.database_entry_start(config)

esm_runscripts.database_actions.database_entry_success(config)

esm_runscripts.esm_batch_system module

exception esm_runscripts.esm_batch_system.UnknownBatchSystemError

Bases: Exception

Raise this exception when an unknown batch system is encountered

class esm_runscripts.esm_batch_system.esm_batch_system(config, name)

Bases: object

calc_requirements(config)

static calculate_requirements(config)

check_if_submitted()

static get_batch_header(config)

static get_environment(config)

get_job_state(jobid)

get_jobid()

static get_run_commands(config)

static get_sad_filename(config)

static get_submit_command(config, sadfilename)

job_is_still_running(jobid)

static submit(config)

static write_simple_runscript(config)

esm_runscripts.esm_coupler module

class esm_runscripts.esm_coupler.esm_coupler(full_config, name)

Bases: object

add_couplings(full_config)

add_files(full_config)

finalize(destination_dir)

prepare(full_config, destination_dir)

prepare_restarts(full_config)

print_config_files()

tidy(full_config)

esm_runscripts.esm_methods module

esm_runscripts.esm_methods.set_global_attr(fname, attribute, value)

esm_runscripts.esm_sim_objects module

Documentation goes here

class esm_runscripts.esm_sim_objects.SimulationSetup(command_line_config=None, user_config=None)

Bases: object

add_submission_info()

assemble_error_list()

assemble_file_lists()

check_for_errors(error_check_list, time, monitor_file)

compute(kill_after_submit=True)

All steps needed for a model computation.

Parameters:kill_after_submit (bool) – Default True. If set, the entire Python instance is killed with a sys.exit() as the very last after job submission.

copy_all_results_to_exp()

end_it_all()

finalize_file_lists(filetypes)

get_total_config_from_user_config(user_config)

get_user_config_from_command_line(command_line_config)

init_coupler()

initialize_batch_system()

job_is_still_running()

static merge_thisrun_into_experiment(config)

postprocess()

set_prev_date()

Sets several variables relevant for the previous date. Loops over all models in valid_model_names, and sets model variables for: * prev_date * parent_expid * parent_date * parent_restart_dir

tidy()

wait_and_observe(monitor_file)

esm_runscripts.esm_sim_objects.date_representer(dumper, date)

esm_runscripts.filelists module

esm_runscripts.filelists.assemble_intermediate_files_and_finalize_targets(config)

esm_runscripts.filelists.check_for_unknown_files(config)

esm_runscripts.filelists.choose_needed_files(config)

esm_runscripts.filelists.complete_restart_in(config)

esm_runscripts.filelists.complete_sources(config)

esm_runscripts.filelists.complete_targets(config)

esm_runscripts.filelists.find_correct_source(mconfig, file_source, year)

esm_runscripts.filelists.globbing(config)

esm_runscripts.filelists.log_used_files(config, filetypes)

esm_runscripts.filelists.rename_sources_to_targets(config)

esm_runscripts.filelists.replace_year_placeholder(config)

esm_runscripts.filelists.target_subfolders(config)

esm_runscripts.jobclass module

class esm_runscripts.jobclass.jobclass(job_type, recipe_steps=None)

Bases: object

assemble_file_lists(config, filetypes)

static assemble_log_message(config, message, message_sep=None, timestampStr_from_Unix=False)

Assembles message for log file. See doc for write_to_log

static copy_files(config, flist, source, target)

static copy_files_from_work_to_thisrun(config, target='thisrun', source='work')

static end_it_all(config, silent=False)

evaluate(config)

filetype_specific_dict = {}

find_correct_source(file_source, year)

static print_used_files(config)

really_assemble_file_list(config, model, filetypes)

relevant_files = []

static report_missing_files(config)

static write_to_log(config, message, message_sep=None)

Puts a message into the experiment log file

Parameters:

• message (list) – A list of the message elements; which is joined by either (highest to lowest): 1) the message_sep argument passed to the method, 2) The user’s chosen seperator, as written in self.config["general"]["experiment_log_file_message_sep"], 3) An empty space " ".
• message_sep (None) – The hard-coded message seperator to use; which ignores user choices.

Note

The user can control two things regarding the logfile format:

1. The datestamp formatting, whjich is taken from the config section general.experiment_log_file_dateformat.
2. The message seperators; taken from general.experiment_log_file_message_sep. Note that if the programmer passes a message_sep argument; this one wins over the user choice.

esm_runscripts.namelists module

esm-runscripts Core Plugins for dealing with Fortran Namelists.

Provides plugins for loading, modifying, deleting, and writing Fortran Namelists as part of the esm-runscripts recipe. All plugins are found under the class Namelist as static methods. A deprecated class namelist (small “n”) is provided, which warns you when it is used.

class esm_runscripts.namelists.Namelist

Bases: object

Methods for dealing with FORTRAN namelists

static nmls_finalize(mconfig)

Writes namelists to disk after all modifications have finished.

User Information

Part of the main log output will be a section specifing the actual namelists that have been used for your simulation, including all relevant additions, removals, or changes.

Programmer Information

A copy of the f90nml object representations of the namelists is stored under the dictionary key “namelist_objs”, as a dictionary of (“namelist_name”, f90nml_objfect) key/value pairs.

Warning

Removing this step from your recipe might result in a broken run, as the namelists will not be present in their desired form! Even if your model runs, it might not contain all user-required changes.

Parameters:mconfig (dict) – The model (e.g. ECHAM, FESOM, NEMO or OIFS) configuration Returns:mconfig – The modified configuration. Return type:dict

static nmls_load(mconfig)

Loads Fortran namelists into the configuration dictionary.

User Information

To associate namelists with a specific model, you should have a section in your configuration that lists the namelists:

fesom:
    namelists:
        - "namelist.config"
        - "namelist.oce"
        - "namelist.ice"
        - "namelist.diag"

Programmer Information

The namelists are represented by f90nml Namelist objects, and are stored under:

mconfig["namelists"]["namelist.echam"]``

This would point to the ECHAM namelist as a f90nml object, which closely resembles a dictionary.

The actual namelists to load are listed in the raw configuration as a list of strings:

mconfig['namelists'] = ['nml1', 'nml2', 'nml3', ...]

Namelists are assumed to have been copied to mconfig["thisrun_config_dir"], and are loaded from there.

If the mconfig has a key "namelist_case" equal to “uppercase”, the uppercase attribute of the f90nml representation of the namelist is set to True.

Parameters:mconfig (dict) – The model (e.g. ECHAM, FESOM, NEMO or OIFS) configuration Returns:mconfig – The modified configuration. Return type:dict

static nmls_modify(mconfig)

Performs namelist changes.

User Information

In the configuration file, you should have a section as:

echam:
    namelist_changes:
        namelist.echam:
            radctl:
                co2vmr: 1200e-6

This would change the value of the echam namelist (namelist.echam), subsection radctl, entry co2vmr to the value 1200e-6.

Programmer Information

IDEA(PG): Maybe we can provide examples of how these functions are used in the code?

Note

Actual changes are performed by the f90nml package patch fuction. See here: https://tinyurl.com/y4ydz363

Parameters:mconfig (dict) – The model (e.g. ECHAM, FESOM, NEMO or OIFS) configuration Returns:mconfig – The modified configuration. Return type:dict

static nmls_remove(mconfig)

Removes an element from a namelist chapter.

User Information

In the configuration file, assume you have:

echam:
    namelist_changes:
        namelist.echam:
            radctl:
                co2vmr: "remove_from_namelist"

In this case, the entry co2vmr would be deleted from the radctl section of namelist.echam.

Programmer Information

IDEA(PG): Maybe we can provide examples of how these functions are used in the code?

Parameters:mconfig (dict) – The model (e.g. ECHAM, FESOM, NEMO or OIFS) configuration Returns:mconfig – The modified configuration. Return type:dict

class esm_runscripts.namelists.namelist(*args, **kwargs)

Bases: esm_runscripts.namelists.Namelist

Legacy class name. Please use Namelist instead!

esm_runscripts.oasis module

class esm_runscripts.oasis.oasis(nb_of_couplings=1, coupled_execs=['echam', 'fesom'], runtime=1, debug_level=1, nnorest='F', mct_version='4.0', lucia=False)

Bases: object

add_coupling(lefts, lgrid, rights, rgrid, direction, transformation, restart_file, time_step, lresume)

add_output_file(lefts, rights, leftmodel, rightmodel, config)

add_restart_files(restart_file, fconfig)

finalize(destination_dir)

prepare_restarts(restart_file, all_fields, model, config)

print_config_files()

esm_runscripts.sadfile module

class esm_runscripts.sadfile.sadfile(config, commands)

Bases: object

static write_simple_runscript(self, write_tidy_call=True)

esm_runscripts.slurm module

Contains functions for dealing with SLURM-based batch systems

class esm_runscripts.slurm.Slurm(config)

Bases: object

Deals with SLURM, allowing you to check if a job is submitted, get the current job ID, generate a srun hostfile, get the current job state, and check if a job is still running.

filename

The filename for srun commands, defaults to hostfile_srun

Type:str

path

Full path to this file, defaults to thisrun_scripts_dir / filename

Type:str

Parameters:config (dict) – The run configuration, needed to determine where the script directory for this particular run is.

calc_requirements(config)

Calculates requirements and writes them to self.path.

static check_if_submitted()

Determines if a job is submitted in the currently running shell by checking for SLURM_JOB_ID in the environment

Returns: Return type:bool

static get_job_state(jobid)

Returns the jobstate full name. See man squeue, section JOB STATE CODES for more details.

Parameters:jobid – str or int. The SLURM job id as displayed in, e.g. squeue Returns:The short job state. Return type:str

static get_jobid()

Gets the current SLURM JOB ID

Returns: Return type:str or None

static job_is_still_running(jobid)

Returns a boolean if the job is still running