From 3afe2c22f8b23489ccff04be2747d5520daf8756 Mon Sep 17 00:00:00 2001 From: Levi Waldron Date: Fri, 15 Jul 2022 14:05:21 +0200 Subject: [PATCH 01/17] bioconductor_full -> bioconductor_docker Bioconductor docker image is now named bioconductor_docker --- docs/settings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/settings.md b/docs/settings.md index 7242073..25d9582 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -16,7 +16,7 @@ bulker: default: docker_args: -v /project/shefflab/database/redis:/data bioconductor: - bioconductor_full: + bioconductor_docker: default: docker_args: --volume=${HOME}/.local/lib/R:/usr/local/lib/R/host-site-library devel: @@ -25,4 +25,4 @@ bulker: For the `redis` example, we're mounting a custom folder to `/data`, but only on `docker:redis` containers. -For the bioconductor example, we're showing how you can mount different host folders to the same container spot, depending on the *tag* (version) of the image being used. This is useful for separating your development vs. stable R packages, for example. \ No newline at end of file +For the bioconductor example, we're showing how you can mount different host folders to the same container spot, depending on the *tag* (version) of the image being used. This is useful for separating your development vs. stable R packages, for example. From 10cc2fb3b49b4d5f06e386c63b6136fe9083f192 Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Fri, 30 Jun 2023 08:38:29 -0400 Subject: [PATCH 02/17] Rename pytest.yaml to run-pytest.yml rename workflow for databio website integration --- .github/workflows/{pytest.yaml => run-pytest.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{pytest.yaml => run-pytest.yml} (99%) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/run-pytest.yml similarity index 99% rename from .github/workflows/pytest.yaml rename to .github/workflows/run-pytest.yml index 671a557..c82f2cd 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/run-pytest.yml @@ -33,4 +33,4 @@ jobs: - name: Run pytest tests run: pytest - \ No newline at end of file + From 1e06a28a1bb993f55ca2d0186140f87140fa6c04 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:30:09 -0500 Subject: [PATCH 03/17] Drop attmap in support of yacman v1 --- .gitignore | 4 +- bulker/bulker.py | 297 ++++++++++++++-------------- bulker/templates/bulker_config.yaml | 24 +-- tests/test_bulker.py | 27 ++- 4 files changed, 178 insertions(+), 174 deletions(-) diff --git a/.gitignore b/.gitignore index 0bfa31a..ae116d3 100644 --- a/.gitignore +++ b/.gitignore @@ -81,4 +81,6 @@ bulker.egg-info/ docs_jupyter/biobase_example docs_jupyter/templates/ -docs_jupyter/fasterq* \ No newline at end of file +docs_jupyter/fasterq* + +build \ No newline at end of file diff --git a/bulker/bulker.py b/bulker/bulker.py index 91d80b2..0a8def6 100644 --- a/bulker/bulker.py +++ b/bulker/bulker.py @@ -277,13 +277,13 @@ def bulker_envvars_add(bulker_config, variable): _LOGGER.error("You must specify a variable.") return - bulker_config.make_writable() - if variable in bulker_config.bulker.envvars: - _LOGGER.info("Variable '{}' already present".format(variable)) - else: - _LOGGER.info("Adding variable '{}'".format(variable)) - bulker_config.bulker.envvars.append(variable) - bulker_config.write() + with bulker_config as bcfg: + if variable in bcfg["bulker"]["envvars"]: + _LOGGER.info("Variable '{}' already present".format(variable)) + else: + _LOGGER.info("Adding variable '{}'".format(variable)) + bcfg["bulker"]["envvars"].append(variable) + bcfg.write() def bulker_envvars_remove(bulker_config, variable): """ @@ -297,13 +297,14 @@ def bulker_envvars_remove(bulker_config, variable): _LOGGER.error("You must specify a variable.") return - bulker_config.make_writable() - if variable in bulker_config.bulker.envvars: - _LOGGER.info("Removing variable '{}'".format(variable)) - bulker_config.bulker.envvars.remove(variable) - else: - _LOGGER.info("Variable not found '{}'".format(variable)) - bulker_config.write() + with bulker_config as bcfg: + bcfg.make_writable() + if variable in bcfg["bulker"]["envvars"]: + _LOGGER.info("Removing variable '{}'".format(variable)) + bcfg["bulker"]["envvars"].remove(variable) + else: + _LOGGER.info("Variable not found '{}'".format(variable)) + bcfg.write() def bulker_init(config_path, template_config_path, container_engine=None): @@ -339,20 +340,20 @@ def bulker_init(config_path, template_config_path, container_engine=None): # templates_subdir = TEMPLATE_SUBDIR copy_tree(os.path.dirname(template_config_path), dest_templates_dir) new_template = os.path.join(dest_folder, os.path.basename(template_config_path)) - bulker_config = yacman.YacAttMap(filepath=template_config_path, writable=False, skip_read_lock=True) + bulker_config = yacman.YAMLConfigManager(filepath=template_config_path, locked=False, skip_read_lock=True) _LOGGER.debug("Engine used: {}".format(container_engine)) - bulker_config.bulker.container_engine = container_engine - if bulker_config.bulker.container_engine == "docker": - bulker_config.bulker.executable_template = os.path.join(TEMPLATE_SUBDIR, DOCKER_EXE_TEMPLATE) - bulker_config.bulker.shell_template = os.path.join(TEMPLATE_SUBDIR, DOCKER_SHELL_TEMPLATE) - bulker_config.bulker.build_template = os.path.join(TEMPLATE_SUBDIR, DOCKER_BUILD_TEMPLATE) - elif bulker_config.bulker.container_engine == "singularity": - bulker_config.bulker.executable_template = os.path.join(TEMPLATE_SUBDIR, SINGULARITY_EXE_TEMPLATE) - bulker_config.bulker.shell_template = os.path.join(TEMPLATE_SUBDIR, SINGULARITY_SHELL_TEMPLATE) - bulker_config.bulker.build_template = os.path.join(TEMPLATE_SUBDIR, SINGULARITY_BUILD_TEMPLATE) - bulker_config.bulker.rcfile = os.path.join(TEMPLATE_SUBDIR, RCFILE_TEMPLATE) - bulker_config.bulker.rcfile_strict = os.path.join(TEMPLATE_SUBDIR, RCFILE_STRICT_TEMPLATE) - bulker_config.write(config_path) + bulker_config["bulker"]["container_engine"] = container_engine + if bulker_config["bulker"]["container_engine"] == "docker": + bulker_config["bulker"]["executable_template"] = os.path.join(TEMPLATE_SUBDIR, DOCKER_EXE_TEMPLATE) + bulker_config["bulker"]["shell_template"] = os.path.join(TEMPLATE_SUBDIR, DOCKER_SHELL_TEMPLATE) + bulker_config["bulker"]["build_template"] = os.path.join(TEMPLATE_SUBDIR, DOCKER_BUILD_TEMPLATE) + elif bulker_config["bulker"]["container_engine"] == "singularity": + bulker_config["bulker"]["executable_template"] = os.path.join(TEMPLATE_SUBDIR, SINGULARITY_EXE_TEMPLATE) + bulker_config["bulker"]["shell_template"] = os.path.join(TEMPLATE_SUBDIR, SINGULARITY_SHELL_TEMPLATE) + bulker_config["bulker"]["build_template"] = os.path.join(TEMPLATE_SUBDIR, SINGULARITY_BUILD_TEMPLATE) + bulker_config["bulker"]["rcfile"] = os.path.join(TEMPLATE_SUBDIR, RCFILE_TEMPLATE) + bulker_config["bulker"]["rcfile_strict"] = os.path.join(TEMPLATE_SUBDIR, RCFILE_STRICT_TEMPLATE) + bulker_config.write_copy(config_path) # copyfile(template_config_path, new_template) # os.rename(new_template, config_path) _LOGGER.info("Wrote new configuration file: {}".format(config_path)) @@ -380,8 +381,8 @@ def get_imports(manifest, bcfg, recurse=False): imports = set() if not manifest: return imports - if hasattr(manifest.manifest, "imports") and manifest.manifest.imports: - for imp in manifest.manifest.imports: + if "imports" in manifest["manifest"] and manifest["manifest"]["imports"]: + for imp in manifest["manifest"]["imports"]: imp_manifest, imp_cratevars = load_remote_registry_path(bcfg, imp, None) imports.add(imp) @@ -399,8 +400,8 @@ def bulker_reload(bcfg): all_manifests_to_load = set() _LOGGER.info("Recursively identifying all loaded manifests...") - if bcfg.bulker.crates: - for namespace, crates in bcfg.bulker.crates.items(): + if bcfg["bulker"]["crates"]: + for namespace, crates in bcfg["bulker"]["crates"].items(): for crate, tags in crates.items(): for tag, path in tags.items(): crate_registry_path = fmt.format(namespace=namespace, crate=crate, @@ -438,35 +439,35 @@ def bulker_load(manifest, cratevars, bcfg, exe_jinja2_template, manifest_name = cratevars['crate'] # We store them in folder: namespace/crate/version if not crate_path: - crate_path = os.path.join(bcfg.bulker.default_crate_folder, + crate_path = os.path.join(bcfg["bulker"]["default_crate_folder"], cratevars['namespace'], manifest_name, cratevars['tag']) if not os.path.isabs(crate_path): - crate_path = os.path.join(os.path.dirname(bcfg["__internal"].file_path), crate_path) + crate_path = os.path.join(os.path.dirname(bcfg.filepath), crate_path) _LOGGER.debug("Crate path: {}".format(crate_path)) _LOGGER.debug("cratevars: {}".format(cratevars)) # Update the config file - if not bcfg.bulker.crates: - bcfg.bulker.crates = {} - if not hasattr(bcfg.bulker.crates, cratevars['namespace']): - bcfg.bulker.crates[cratevars['namespace']] = yacman.YacAttMap({}) - if not hasattr(bcfg.bulker.crates[cratevars['namespace']], cratevars['crate']): - bcfg.bulker.crates[cratevars['namespace']][cratevars['crate']] = yacman.YacAttMap({}) - if hasattr(bcfg.bulker.crates[cratevars['namespace']][cratevars['crate']], cratevars['tag']): - _LOGGER.debug(bcfg.bulker.crates[cratevars['namespace']][cratevars['crate']].to_dict()) + if not bcfg["bulker"]["crates"]: + bcfg["bulker"]["crates"] = {} + if not cratevars['namespace'] in bcfg["bulker"]["crates"]: + bcfg["bulker"]["crates"][cratevars['namespace']] = yacman.YAMLConfigManager({}) + if not cratevars['crate'] in bcfg["bulker"]["crates"][cratevars['namespace']]: + bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']] = yacman.YAMLConfigManager({}) + if cratevars['tag'] in bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']]: + _LOGGER.debug(bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']]) if not (force or query_yes_no("That manifest has already been loaded. Overwrite?")): return else: - bcfg.bulker.crates[cratevars['namespace']][cratevars['crate']][str(cratevars['tag'])] = crate_path + bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']][str(cratevars['tag'])] = crate_path _LOGGER.warning("Removing all executables in: {}".format(crate_path)) try: shutil.rmtree(crate_path) except: _LOGGER.error("Error removing crate at {}. Did your crate path change? Remove it manually.".format(crate_path)) else: - bcfg.bulker.crates[cratevars['namespace']][cratevars['crate']][str(cratevars['tag'])] = crate_path + bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']][str(cratevars['tag'])] = crate_path # Now make the crate @@ -478,12 +479,12 @@ def bulker_load(manifest, cratevars, bcfg, exe_jinja2_template, for imp in imps: reload_import = recurse imp_cratevars = parse_registry_path(imp) - imp_crate_path = os.path.join(bcfg.bulker.default_crate_folder, + imp_crate_path = os.path.join(bcfg["bulker"]["default_crate_folder"], imp_cratevars['namespace'], imp_cratevars['crate'], imp_cratevars['tag']) if not os.path.isabs(imp_crate_path): - imp_crate_path = os.path.join(os.path.dirname(bcfg["__internal"].file_path), imp_crate_path) + imp_crate_path = os.path.join(os.path.dirname(bcfg.filepath), imp_crate_path) if not os.path.exists(imp_crate_path): _LOGGER.error("Nonexistent crate: '{}' from '{}'. Reloading...".format(imp, imp_crate_path)) reload_import = True @@ -506,11 +507,11 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): imvars = parse_registry_path_image(pkg['docker_image']) _LOGGER.debug(imvars) try: - amap = bcfg.bulker.tool_args[imvars['namespace']][imvars['image']] - if imvars['tag'] != 'default' and hasattr(amap, imvars['tag']): + amap = bcfg["bulker"]["tool_args"][imvars['namespace']][imvars['image']] + if imvars['tag'] != 'default' and imvars['tag'] in amap: string = amap[imvars['tag']][hosttool_arg_key] else: - string = amap.default[hosttool_arg_key] + string = amap["default"][hosttool_arg_key] _LOGGER.debug(string) return string except: @@ -519,21 +520,21 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): cmdlist = [] cmd_count = 0 - if hasattr(manifest.manifest, "commands") and manifest.manifest.commands: - for pkg in manifest.manifest.commands: + if "commands" in manifest["manifest"] and manifest["manifest"]["commands"]: + for pkg in manifest["manifest"]["commands"]: _LOGGER.debug(pkg) - pkg.update(bcfg.bulker) # Add terms from the bulker config - pkg = copy.deepcopy(yacman.YacAttMap(pkg)) # (otherwise it's just a dict) + pkg.update(bcfg["bulker"]) # Add terms from the bulker config + pkg = copy.deepcopy(yacman.YAMLConfigManager(pkg)) # (otherwise it's just a dict) # We have to deepcopy it so that changes we make to pkg aren't reflected in bcfg. - if pkg.container_engine == "singularity" and "singularity_image_folder" in pkg: + if pkg["container_engine"] == "singularity" and "singularity_image_folder" in pkg: pkg["singularity_image"] = os.path.basename(pkg["docker_image"]) pkg["namespace"] = os.path.dirname(pkg["docker_image"]) if os.path.isabs(pkg["singularity_image_folder"]): sif = pkg["singularity_image_folder"] else: - sif = os.path.join(os.path.dirname(bcfg["__internal"].file_path), + sif = os.path.join(os.path.dirname(bcfg.filepath), pkg["singularity_image_folder"]) pkg["singularity_fullpath"] = os.path.join( @@ -548,10 +549,10 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): cmdlist.append(command) # Add any host-specific tool-specific args - hosttool_arg_key = "{engine}_args".format(engine=bcfg.bulker.container_engine) + hosttool_arg_key = "{engine}_args".format(engine=bcfg["bulker"]["container_engine"]) hts = host_tool_specific_args(bcfg, pkg, hosttool_arg_key) _LOGGER.debug("Adding host-tool args: {}".format(hts)) - if hasattr(pkg, hosttool_arg_key): + if hosttool_arg_key in pkg: pkg[hosttool_arg_key] += " " + hts else: pkg[hosttool_arg_key] = hts @@ -591,16 +592,16 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): _LOGGER.error("------ Error building. Build script used: ------") _LOGGER.error(buildscript) _LOGGER.error("------------------------------------------------") - if pkg.container_engine == "singularity": + if pkg["container_engine"] == "singularity": _LOGGER.info("Image available at: {cmd}".format(cmd=pkg["singularity_fullpath"])) else: - _LOGGER.info("Docker image available as: {cmd}".format(cmd=pkg.docker_image)) + _LOGGER.info("Docker image available as: {cmd}".format(cmd=pkg["docker_image"])) # host commands host_cmdlist = [] - if hasattr(manifest.manifest, "host_commands") and manifest.manifest.host_commands: + if "host_commands" in manifest["manifest"] and manifest["manifest"]["host_commands"]: _LOGGER.info("Populating host commands") - for cmd in manifest.manifest.host_commands: + for cmd in manifest["manifest"]["host_commands"]: _LOGGER.debug(cmd) if not is_command_callable(cmd): _LOGGER.warning("Requested host command is not callable and " @@ -646,14 +647,14 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): _LOGGER.info("Host commands available: {}".format(", ".join(host_cmdlist))) - - bcfg.write() + with bcfg as bcfg: + bcfg.write() def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=True): """ Activates a given crate. - :param yacman.YacAttMap bulker_config: The bulker configuration object. + :param yacman.YAMLConfigManager bulker_config: The bulker configuration object. :param list cratelist: a list of cratevars objects, which are dicts with values for 'namespace', 'crate', and 'tag'. :param bool echo: Should we just echo the new PATH to create? Otherwise, the @@ -668,8 +669,8 @@ def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=T new_env = os.environ - if hasattr(bulker_config.bulker, "shell_path"): - shellpath = os.path.expandvars(bulker_config.bulker.shell_path) + if "shell_path" in bulker_config["bulker"]: + shellpath = os.path.expandvars(bulker_config["bulker"]["shell_path"]) else: shellpath = os.path.expandvars("$SHELL") @@ -679,8 +680,8 @@ def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=T shell_list = [bashpath, bashpath] - if hasattr(bulker_config.bulker, "shell_rc"): - shell_rc = os.path.expandvars(bulker_config.bulker.shell_rc) + if "shell_rc" in bulker_config["bulker"]: + shell_rc = os.path.expandvars(bulker_config["bulker"]["shell_rc"]) else: if os.path.basename(shellpath) == "bash": shell_rc = "$HOME/.bashrc" @@ -710,8 +711,8 @@ def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=T _LOGGER.debug("Newpath: {}".format(newpath)) - if hasattr(bulker_config.bulker, "shell_prompt"): - ps1 = bulker_config.bulker.shell_prompt + if "shell_prompt" in bulker_config["bulker"]: + ps1 = bulker_config["bulker"]["shell_prompt"] else: if os.path.basename(shellpath) == "bash": ps1 = "\\u@\\b:\\w\\a\\$ " @@ -750,16 +751,16 @@ def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=T new_env["BULKERSHELLRC"] = shell_rc if strict: - for k in bulker_config.bulker.envvars: + for k in bulker_config["bulker"]["envvars"]: new_env[k] = os.environ.get(k, "") if os.path.basename(shellpath) == "bash": if strict: - rcfile = mkabs(bulker_config.bulker.rcfile_strict, - os.path.dirname(bulker_config["__internal"].file_path)) + rcfile = mkabs(bulker_config["bulker"]["rcfile_strict"], + os.path.dirname(bulker_config.filepath)) else: - rcfile = mkabs(bulker_config.bulker.rcfile, - os.path.dirname(bulker_config["__internal"].file_path)) + rcfile = mkabs(bulker_config["bulker"]["rcfile"], + os.path.dirname(bulker_config.filepath)) shell_list.append("--rcfile") shell_list.append(rcfile) @@ -769,12 +770,12 @@ def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=T if os.path.basename(shellpath) == "zsh": if strict: rcfolder = mkabs(os.path.join( - os.path.dirname(bulker_config.bulker.rcfile_strict), - "zsh_start_strict"), os.path.dirname(bulker_config["__internal"].file_path)) + os.path.dirname(bulker_config["bulker"]["rcfile_strict"]), + "zsh_start_strict"), os.path.dirname(bulker_config.filepath)) else: rcfolder = mkabs(os.path.join( - os.path.dirname(bulker_config.bulker.rcfile_strict), - "zsh_start"), os.path.dirname(bulker_config["__internal"].file_path)) + os.path.dirname(bulker_config["bulker"]["rcfile_strict"]), + "zsh_start"), os.path.dirname(bulker_config.filepath)) new_env["ZDOTDIR"] = rcfolder _LOGGER.debug("ZDOTDIR: {}".format(new_env["ZDOTDIR"])) @@ -790,13 +791,13 @@ def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=T def get_local_path(bulker_config, cratevars): """ :param dict cratevars: dict with crate metadata returned from parse_registry_path - :param YacAttMap bulker_config: bulker config object + :param YAMLConfigManager bulker_config: bulker config object :return str: path to requested crate folder """ _LOGGER.debug(cratevars) - _LOGGER.debug(bulker_config.bulker.crates[cratevars["namespace"]][cratevars["crate"]].to_dict()) + _LOGGER.debug(bulker_config["bulker"]["crates"][cratevars["namespace"]][cratevars["crate"]]) - return bulker_config.bulker.crates[cratevars["namespace"]][cratevars["crate"]][cratevars["tag"]] + return bulker_config["bulker"]["crates"][cratevars["namespace"]][cratevars["crate"]][cratevars["tag"]] def get_new_PATH(bulker_config, cratelist, strict=False): """ @@ -804,7 +805,7 @@ def get_new_PATH(bulker_config, cratelist, strict=False): :: param str crates :: string with a comma-separated list of crate identifiers """ - if not bulker_config.bulker.crates: + if not bulker_config["bulker"]["crates"]: raise MissingCrateError("No crates exist") cratepaths = "" @@ -965,8 +966,8 @@ def load_remote_registry_path(bulker_config, registry_path, filepath=None): cratevars = parse_registry_path(registry_path) if cratevars: # assemble the query string - if 'registry_url' in bulker_config.bulker: - base_url = bulker_config.bulker.registry_url + if "registry_url" in bulker_config["bulker"]: + base_url = bulker_config["bulker"]["registry_url"] else: # base_url = "http://big.databio.org/bulker/" base_url = DEFAULT_BASE_URL @@ -1005,9 +1006,9 @@ def load_remote_registry_path(bulker_config, registry_path, filepath=None): raise Exception("No remote manifest found") data = response.read() # a `bytes` object text = data.decode('utf-8') - manifest_lines = yacman.YacAttMap(yamldata=text) + manifest_lines = yacman.YAMLConfigManager(yamldata=text) else: - manifest_lines = yacman.YacAttMap(filepath=filepath) + manifest_lines = yacman.YAMLConfigManager(filepath=filepath) return manifest_lines, cratevars @@ -1047,13 +1048,13 @@ def prep_load(bulker_config, crate_registry_paths, manifest=None, build=False): build_template_jinja = None shell_template_jinja = None - exe_template = mkabs(bulker_config.bulker.executable_template, - os.path.dirname(bulker_config["__internal"].file_path)) - build_template = mkabs(bulker_config.bulker.build_template, - os.path.dirname(bulker_config["__internal"].file_path)) + exe_template = mkabs(bulker_config["bulker"]["executable_template"], + os.path.dirname(bulker_config.filepath)) + build_template = mkabs(bulker_config["bulker"]["build_template"], + os.path.dirname(bulker_config.filepath)) try: - shell_template = mkabs(bulker_config.bulker.shell_template, - os.path.dirname(bulker_config["__internal"].file_path)) + shell_template = mkabs(bulker_config["bulker"]["shell_template"], + os.path.dirname(bulker_config.filepath)) except AttributeError: _LOGGER.error("You need to re-initialize your bulker config or add a 'shell_template' attribute.") sys.exit(1) @@ -1102,7 +1103,7 @@ def parse_cwl(cwl_file): """ :param str cwl_file: CWL tool description file. """ - yam = yacman.YacAttMap(filepath=cwl_file) + yam = yacman.YAMLConfigManager(filepath=cwl_file) if yam["class"] != "CommandLineTool": _LOGGER.info("CWL file of wrong class: {} ({})".format(cwl_file, yam["class"])) return None @@ -1125,18 +1126,18 @@ def parse_cwl(cwl_file): try: image = None - if hasattr(yam, "requirements"): - if hasattr(yam.requirements, "DockerRequirement"): - image = yam.requirements.DockerRequirement.dockerPull - elif isinstance(yam.requirements, list): - for req in yam.requirements: + if "requirements" in yam: + if "DockerRequirement" in yam["requirements"]: + image = yam["requirements"]["DockerRequirement"]["dockerPull"] + elif isinstance(yam["requirements"], list): + for req in yam["requirements"]: if req["class"] == "DockerRequirement": image = req["dockerPull"] - if not image and hasattr(yam, "hints"): - if hasattr(yam.hints, "DockerRequirement"): - image = yam.hints.DockerRequirement.dockerPull - elif isinstance(yam.hints, list): - for hint in yam.hints: + if not image and "hints" in yam: + if "DockerRequirement" in yam["hints"]: + image = yam["hints"]["DockerRequirement"]["dockerPull"] + elif isinstance(yam["hints"], list): + for hint in yam["hints"]: if hint["class"] == "DockerRequirement": image = hint["dockerPull"] if not image: @@ -1152,7 +1153,7 @@ def parse_cwl(cwl_file): if str(image).startswith("$include"): print(str(image)) - x = yacman.YacAttMap(yamldata=str(image)) + x = yacman.YAMLConfigManager(yamldata=str(image)) file_path = str(x["$include"]) with open(os.path.join(os.path.dirname(cwl_file), file_path), 'r') as f: contents = f.read() @@ -1169,36 +1170,37 @@ def parse_cwl(cwl_file): def bulker_unload(bulker_config, crate_registry_paths): cratelist = parse_registry_paths(crate_registry_paths, - bulker_config.bulker.default_namespace) + bulker_config["bulker"]["default_namespace"]) _LOGGER.info("Unloading crates: {}".format(crate_registry_paths)) removed_crates = [] for cratemeta in cratelist: namespace = cratemeta['namespace'] - if namespace in bulker_config.bulker.crates: + if namespace in bulker_config["bulker"]["crates"]: crate = cratemeta['crate'] - # print(bulker_config.bulker.crates[namespace]) - if crate in bulker_config.bulker.crates[namespace]: + # print(bulker_config["bulker"].crates[namespace]) + if crate in bulker_config["bulker"]["crates"][namespace]: tag = cratemeta['tag'] - # print(bulker_config.bulker.crates[namespace][crate]) - if tag in bulker_config.bulker.crates[namespace][crate]: + # print(bulker_config["bulker"].crates[namespace][crate]) + if tag in bulker_config["bulker"]["crates"][namespace][crate]: regpath = "{namespace}/{crate}:{tag}".format( namespace=namespace, crate=crate, tag=tag) _LOGGER.info("Removing crate: '{}'".format(regpath)) bulker_config.make_writable() - # bulker_config.bulker.crates[namespace][crate][tag] = None - crate_path = bulker_config.bulker.crates[namespace][crate][tag] - del bulker_config.bulker.crates[namespace][crate][tag] + # bulker_config["bulker"].crates[namespace][crate][tag] = None + crate_path = bulker_config["bulker"]["crates"][namespace][crate][tag] + del bulker_config["bulker"]["crates"][namespace][crate][tag] try: shutil.rmtree(crate_path) except: _LOGGER.error("Error removing crate at {}. Did your crate path change? Remove it manually.".format(crate_path)) - if len(bulker_config.bulker.crates[namespace][crate]) ==0: + if len(bulker_config["bulker"]["crates"][namespace][crate]) ==0: _LOGGER.info("Last tag!") - del bulker_config.bulker.crates[namespace][crate] - bulker_config.write() + del bulker_config["bulker"]["crates"][namespace][crate] + with bulker_config as bcfg: + bcfg.write() removed_crates.append(regpath) if len(removed_crates) > 0: @@ -1232,9 +1234,9 @@ def main(): sys.exit(0) if args.command == "cwl2man": - bm = yacman.YacAttMap() - bm.manifest = yacman.YacAttMap() - bm.manifest.commands = [] + bm = yacman.YAMLConfigManager() + bm.manifest = yacman.YAMLConfigManager() + bm.manifest["commands"] = [] baseCommandsNotFound = [] imagesNotFound = [] @@ -1242,26 +1244,27 @@ def main(): try: cmd = parse_cwl(cwl_file) if cmd: - bm.manifest.commands.append(cmd) + bm.manifest["commands"].append(cmd) except BaseCommandNotFoundException as e: baseCommandsNotFound.append(e.file) except ImageNotFoundException as e: imagesNotFound.append(e.file) - _LOGGER.info("Commands added: {}".format(len(bm.manifest.commands))) + _LOGGER.info("Commands added: {}".format(len(bm.manifest["commands"]))) if len(baseCommandsNotFound) > 0: _LOGGER.info("Command not found ({}): {}".format( len(baseCommandsNotFound), baseCommandsNotFound)) if len(imagesNotFound) > 0: _LOGGER.info("Image not found ({}): {}".format( len(imagesNotFound), imagesNotFound)) - bm.write(args.manifest) + + bm.write_copy(args.manifest) sys.exit(0) # Any remaining commands require config so we process it now. bulkercfg = select_bulker_config(args.config) - bulker_config = yacman.YacAttMap(filepath=bulkercfg, writable=False) + bulker_config = yacman.YAMLConfigManager(filepath=bulkercfg, locked=False) # _LOGGER.info("Bulker config: {}".format(bulkercfg)) if args.command == "envvars": @@ -1273,7 +1276,7 @@ def main(): _LOGGER.debug("Removing env var") _is_writable(os.path.dirname(bulkercfg), check_exist=False) bulker_envvars_remove(bulker_config, args.remove) - _LOGGER.info("Envvars list: {}".format(bulker_config.bulker.envvars)) + _LOGGER.info("Envvars list: {}".format(bulker_config["bulker"].envvars)) sys.exit(0) @@ -1285,12 +1288,12 @@ def main(): args.crate_registry_paths, None) manifest_name = cratevars['crate'] - crate_path = os.path.join(bulker_config.bulker.default_crate_folder, + crate_path = os.path.join(bulker_config["bulker"]["default_crate_folder"], cratevars['namespace'], manifest_name, cratevars['tag']) if not os.path.isabs(crate_path): - crate_path = os.path.join(os.path.dirname(bulker_config["__internal"].file_path), crate_path) + crate_path = os.path.join(os.path.dirname(bulker_config.filepath), crate_path) print("Crate path: {}".format(crate_path)) @@ -1309,8 +1312,8 @@ def main(): if args.simple: fmt = "{namespace}/{crate}:{tag}" crateslist = [] - if bulker_config.bulker.crates: - for namespace, crates in bulker_config.bulker.crates.items(): + if bulker_config["bulker"]["crates"]: + for namespace, crates in bulker_config["bulker"]["crates"].items(): for crate, tags in crates.items(): for tag, path in tags.items(): crateslist.append(fmt.format(namespace=namespace, crate=crate, @@ -1322,8 +1325,8 @@ def main(): _LOGGER.info("Available crates:") fmt = "{namespace}/{crate}:{tag} -- {path}" - if bulker_config.bulker.crates: - for namespace, crates in bulker_config.bulker.crates.items(): + if bulker_config["bulker"].crates: + for namespace, crates in bulker_config["bulker"]["crates"].items(): for crate, tags in crates.items(): for tag, path in tags.items(): print(fmt.format(namespace=namespace, crate=crate, @@ -1337,7 +1340,7 @@ def main(): if args.command == "activate": try: cratelist = parse_registry_paths(args.crate_registry_paths, - bulker_config.bulker.default_namespace) + bulker_config["bulker"]["default_namespace"]) _LOGGER.debug(cratelist) _LOGGER.info("Activating bulker crate: {}{}".format(args.crate_registry_paths, " (Strict)" if args.strict else "")) bulker_activate(bulker_config, cratelist, echo=args.echo, strict=args.strict, prompt=args.no_prompt) @@ -1366,27 +1369,27 @@ def main(): sys.exit(1) if args.command == "load": - bulker_config.make_writable() - manifest, cratevars, exe_template_jinja, shell_template_jinja, build_template_jinja = prep_load( - bulker_config, args.crate_registry_paths, args.manifest, args.build) + with bulker_config as _: + manifest, cratevars, exe_template_jinja, shell_template_jinja, build_template_jinja = prep_load( + bulker_config, args.crate_registry_paths, args.manifest, args.build) - try: - bulker_load(manifest, cratevars, bulker_config, - exe_jinja2_template=exe_template_jinja, - shell_jinja2_template=shell_template_jinja, - crate_path=args.path, - build=build_template_jinja, - force=args.force, - recurse=args.recurse) - except Exception as e: - print(f'Bulker load failed: {e}') - sys.exit(1) + try: + bulker_load(manifest, cratevars, bulker_config, + exe_jinja2_template=exe_template_jinja, + shell_jinja2_template=shell_template_jinja, + crate_path=args.path, + build=build_template_jinja, + force=args.force, + recurse=args.recurse) + except Exception as e: + print(f'Bulker load failed: {e}') + sys.exit(1) if args.command == "reload": - bulker_config.make_writable() _LOGGER.info("Reloading all manifests") - bulker_reload(bulker_config) + with bulker_config as _: + bulker_reload(bulker_config) if args.command == "unload": bulker_unload(bulker_config, args.crate_registry_paths) diff --git a/bulker/templates/bulker_config.yaml b/bulker/templates/bulker_config.yaml index 533adb6..90dc1ea 100644 --- a/bulker/templates/bulker_config.yaml +++ b/bulker/templates/bulker_config.yaml @@ -1,16 +1,18 @@ bulker: - volumes: ['$HOME'] - envvars: ['DISPLAY'] + build_template: templates/docker_build.jinja2 + container_engine: docker + crates: null + default_crate_folder: ${HOME}/bulker_crates + default_namespace: bulker + envvars: + - DISPLAY + executable_template: templates/docker_executable.jinja2 + rcfile: templates/start.sh + rcfile_strict: templates/start_strict.sh registry_url: http://hub.bulker.io/ shell_path: ${SHELL} shell_rc: $HOME/.bashrc - rcfile: start.sh - rcfile_strict: start_strict.sh - default_crate_folder: ${HOME}/bulker_crates + shell_template: templates/docker_shell.jinja2 singularity_image_folder: ${HOME}/simages - container_engine: docker - default_namespace: bulker - executable_template: docker_executable.jinja2 - shell_template: docker_shell.jinja2 - build_template: docker_build.jinja2 - crates: null + volumes: + - $HOME diff --git a/tests/test_bulker.py b/tests/test_bulker.py index 6912098..885569b 100644 --- a/tests/test_bulker.py +++ b/tests/test_bulker.py @@ -16,9 +16,9 @@ def test_yacman(): - bc = yacman.YacAttMap(filepath=DEFAULT_CONFIG_FILEPATH) + bc = yacman.YAMLConfigManager(filepath=DEFAULT_CONFIG_FILEPATH) bc - bc.bulker.default_crate_folder + bc["bulker"]["default_crate_folder"] yaml_str = """\ --- @@ -28,7 +28,7 @@ def test_yacman(): def test_float_idx(): - data = yacman.YacAttMap(yamldata=yaml_str) + data = yacman.YAMLConfigManager(yamldata=yaml_str) # We should be able to access this by string, not by int index. assert(data['2'] == "two") with pytest.raises(KeyError): @@ -51,7 +51,7 @@ def test_bulker_init(): pass bulker_init(DUMMY_CFG_FILEPATH, DEFAULT_CONFIG_FILEPATH, "docker") - bulker_config = yacman.YacAttMap(filepath=DEFAULT_CONFIG_FILEPATH) + bulker_config = yacman.YAMLConfigManager(filepath=DEFAULT_CONFIG_FILEPATH) manifest, cratevars = load_remote_registry_path(bulker_config, "demo", @@ -66,7 +66,7 @@ def test_bulker_init(): def test_bulker_activate(): - bulker_config = yacman.YacAttMap(filepath=DEFAULT_CONFIG_FILEPATH) + bulker_config = yacman.YAMLConfigManager(filepath=DEFAULT_CONFIG_FILEPATH) def test_nonconfig_load(): @@ -85,23 +85,22 @@ def test_nonconfig_load(): pass bulker_init(DUMMY_CFG_FILEPATH, DEFAULT_CONFIG_FILEPATH, "docker") - bulker_config = yacman.YacAttMap(filepath=DUMMY_CFG_FILEPATH) + bulker_config = yacman.YAMLConfigManager(filepath=DUMMY_CFG_FILEPATH) # The 'load' command will write the new crate to the config file; # we don't want it to update the template config file, so make a dummy # filepath that we'll delete later. - bulker_config.make_writable() # for testing, use a local crate folder - bulker_config.bulker.default_crate_folder = DUMMY_CFG_CRATE_SUBDIR + bulker_config["bulker"]["default_crate_folder"] = DUMMY_CFG_CRATE_SUBDIR print("Bulker config: {}".format(bulker_config)) manifest, cratevars = load_remote_registry_path(bulker_config, "demo", None) manifest, cratevars = load_remote_registry_path(bulker_config, "demo", None) - exe_template = mkabs(bulker_config.bulker.executable_template, os.path.dirname(bulker_config.__internal.file_path)) - shell_template = mkabs(bulker_config.bulker.shell_template, - os.path.dirname(bulker_config.__internal.file_path)) + exe_template = mkabs(bulker_config["bulker"]["executable_template"], os.path.dirname(bulker_config.filepath)) + shell_template = mkabs(bulker_config["bulker"]["shell_template"], + os.path.dirname(bulker_config.filepath)) import jinja2 with open(exe_template, 'r') as f: contents = f.read() @@ -111,8 +110,6 @@ def test_nonconfig_load(): contents = f.read() shell_template_jinja = jinja2.Template(contents) - - bulker_load(manifest, cratevars, bulker_config, exe_template_jinja, shell_template_jinja, force=True) # bulker_config.make_readonly() # deprecated this line with yacman improvements? @@ -124,7 +121,7 @@ def test_nonconfig_load(): print(cratelist) # Test a reload with already-removed crate - crate_folder = os.path.join(bulker_config.bulker.default_crate_folder, "bulker/demo/default") + crate_folder = os.path.join(bulker_config["bulker"]["default_crate_folder"], "bulker/demo/default") print("removing {}".format(crate_folder)) # shutil.rmtree(crate_folder) bulker_load(manifest, cratevars, bulker_config, exe_template_jinja, @@ -163,5 +160,5 @@ def test_nonconfig_load(): # out, err, exitcode = capture([os.path.expandvars("$HOME/.local/bin/bulker"), "load", "bogusbogus"]) # assert exitcode == 1 -# manifest = yacman.YacAttMap(filepath="/home/ns5bc/code/bulker/demo/demo_manifest.yaml") +# manifest = yacman.YAMLConfigManager(filepath="/home/ns5bc/code/bulker/demo/demo_manifest.yaml") # bc From c37fc89e109cbbae00291b3d1861327d1b9682b1 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:30:52 -0500 Subject: [PATCH 04/17] version bump --- build/lib/bulker/_version.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 build/lib/bulker/_version.py diff --git a/build/lib/bulker/_version.py b/build/lib/bulker/_version.py new file mode 100644 index 0000000..4910b9e --- /dev/null +++ b/build/lib/bulker/_version.py @@ -0,0 +1 @@ +__version__ = "0.7.3" From da5e4f3142950f4040954dfd61a5cf7a2dc3e5fe Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:31:42 -0500 Subject: [PATCH 05/17] version bump --- bulker/_version.py | 2 +- docs/changelog.md | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bulker/_version.py b/bulker/_version.py index 4910b9e..e498a50 100644 --- a/bulker/_version.py +++ b/bulker/_version.py @@ -1 +1 @@ -__version__ = "0.7.3" +__version__ = "0.7.4-dev" diff --git a/docs/changelog.md b/docs/changelog.md index 3054f0b..e48dbd7 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,8 @@ # Changelog +## [0.7.4] -- Unreleased + + ## [0.7.3] -- 2021-12-08 - Fixed a bug that prevented use when installed in non-writable directory - Added ability to use bash autocompletion From 80e34827bd0244c622ed5de892adfe9630953a21 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:34:45 -0500 Subject: [PATCH 06/17] update python test versions --- .github/workflows/run-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index c82f2cd..936b920 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.9] + python-version: ["3.7", "3.11"] os: [ubuntu-latest] steps: From ca844c4eb29baa97d905c96955a9c9fe27a691ee Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:37:45 -0500 Subject: [PATCH 07/17] can we test on python 3.12 yet? --- .github/workflows/run-pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index 936b920..aa898ce 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.7", "3.11"] + python-version: ["3.8", "3.12"] os: [ubuntu-latest] steps: From de785e97814b4f765afa7302c4d1d5a1b9305fee Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:45:33 -0500 Subject: [PATCH 08/17] switch distutils to shutil for py3.12 --- bulker/bulker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bulker/bulker.py b/bulker/bulker.py index 0a8def6..909b727 100644 --- a/bulker/bulker.py +++ b/bulker/bulker.py @@ -13,8 +13,8 @@ import signal import shutil -from distutils.dir_util import copy_tree -from distutils.spawn import find_executable +from shutil import which, copytree + from ubiquerg import is_url, is_command_callable, parse_registry_path as prp, \ query_yes_no @@ -338,7 +338,7 @@ def bulker_init(config_path, template_config_path, container_engine=None): dest_folder = os.path.dirname(config_path) dest_templates_dir = os.path.join(dest_folder, TEMPLATE_SUBDIR) # templates_subdir = TEMPLATE_SUBDIR - copy_tree(os.path.dirname(template_config_path), dest_templates_dir) + copytree(os.path.dirname(template_config_path), dest_templates_dir, dirs_exist_ok=True) new_template = os.path.join(dest_folder, os.path.basename(template_config_path)) bulker_config = yacman.YAMLConfigManager(filepath=template_config_path, locked=False, skip_read_lock=True) _LOGGER.debug("Engine used: {}".format(container_engine)) @@ -498,7 +498,7 @@ def bulker_load(manifest, cratevars, bcfg, exe_jinja2_template, bulker_load(imp_manifest, imp_cratevars, bcfg, exe_jinja2_template, shell_jinja2_template, crate_path=None, build=build, force=force, recurse=False) _LOGGER.info("Importing crate '{}' from '{}'.".format(imp, imp_crate_path)) - copy_tree(imp_crate_path, crate_path) + copytree(imp_crate_path, crate_path, dirs_exist_ok=True) # should put this in a function def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): @@ -607,7 +607,7 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): _LOGGER.warning("Requested host command is not callable and " "therefore not created: '{}'".format(cmd)) continue - local_exe = find_executable(cmd) + local_exe = which(cmd) path = os.path.join(crate_path, cmd) host_cmdlist.append(cmd) try: From 734cfc5dea8a8d68c887c50604b0aece64e6f3cc Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:53:54 -0500 Subject: [PATCH 09/17] update python versions --- setup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9f09691..3aa5cb2 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,11 @@ classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Scientific/Engineering :: Bio-Informatics" ], keywords="docker, containers, reproducibility, bioinformatics, workflow", From 5427cac07b320bab98b7999960aaa8d5f5e75fb1 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 12 Dec 2023 12:55:42 -0500 Subject: [PATCH 10/17] changelog --- bulker/_version.py | 2 +- docs/changelog.md | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bulker/_version.py b/bulker/_version.py index e498a50..b4e087d 100644 --- a/bulker/_version.py +++ b/bulker/_version.py @@ -1 +1 @@ -__version__ = "0.7.4-dev" +__version__ = "0.8.0-dev" diff --git a/docs/changelog.md b/docs/changelog.md index e48dbd7..d01fb26 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,7 +1,9 @@ # Changelog -## [0.7.4] -- Unreleased - +## [0.8.0] -- Unreleased +- Some routine maintenance +- Prepped for yacman v1, dropping attmap support +- Added support for Python 3.12 ## [0.7.3] -- 2021-12-08 - Fixed a bug that prevented use when installed in non-writable directory @@ -9,7 +11,7 @@ ## [0.7.2] -- 2021-06-24 - Update to yacman 0.8.0, fixing references to internal config attributes. -- Add documentation and clarity for the `shell_prompt` option +- Added documentation and clarity for the `shell_prompt` option - Fixed bug with relative crate root folders ## [0.7.1] -- 2021-03-03 From 17322c138e52a2657509b55d89827c6278c7dae6 Mon Sep 17 00:00:00 2001 From: nsheff Date: Fri, 6 Feb 2026 07:30:11 -0500 Subject: [PATCH 11/17] Update help messaging --- bulker/bulker.py | 67 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/bulker/bulker.py b/bulker/bulker.py index 909b727..a4d65e9 100644 --- a/bulker/bulker.py +++ b/bulker/bulker.py @@ -97,17 +97,47 @@ def build_argparser(): subparser_messages = { "init": "Initialize a new bulker config file", - "inspect": "View name and list of commands for a crate", - "list": "List available bulker crates", + "inspect": "View commands in a loaded crate", + "list": "List loaded bulker crates", "load": "Load a crate from a manifest", - "unload": "Remove a loaded crate from bulker config", - "reload": "Reload all previously loaded manifests", - "activate": "Activate a crate by adding it to PATH", - "run": "Run a command in a crate", - "envvars": "List, add, or remove environment variables to bulker config", + "unload": "Remove a loaded crate from disk and config", + "reload": "Re-fetch and rebuild all loaded crates from their manifests", + "activate": "Start a new shell with crate commands in PATH", + "run": "Run a single command in a crate environment without starting a shell", + "envvars": "List, add, or remove environment variables in bulker config", "cwl2man": "Build a manifest from cwl tool descriptions" } + subparser_examples = { + "init": " bulker init -c ~/.bulker/bulker_config.yaml\n" + " bulker init -c ~/bulker_config.yaml -e singularity", + "inspect": " bulker inspect # inspect the currently active crate\n" + " bulker inspect bulker/demo\n" + " bulker inspect databio/pepatac:1.0.13", + "list": " bulker list\n" + " bulker list -s # simple format for scripting", + "load": " bulker load bulker/demo\n" + " bulker load databio/pepatac:1.0.13\n" + " bulker load -f bulker/demo # overwrite existing\n" + " bulker load -b bulker/demo # also pull container images\n" + " bulker load -m manifest.yaml my/crate # load from local manifest file", + "unload": " bulker unload bulker/demo\n" + " bulker unload databio/pepatac:1.0.13", + "reload": " bulker reload # reload all crates", + "activate": " bulker activate bulker/demo\n" + " bulker activate databio/pepatac:1.0.13\n" + " bulker activate bulker/demo,bulker/pi # multiple crates\n" + " bulker activate demo # uses default namespace\n" + " bulker activate -s bulker/demo # strict: only crate commands in PATH\n" + " bulker activate -e bulker/demo # print exports instead of launching shell", + "run": " bulker run bulker/demo cowsay hello\n" + " bulker run databio/pepatac:1.0.13 samtools --version\n" + " bulker run -s bulker/demo cowsay hi # strict: only crate commands in PATH", + "envvars": " bulker envvars # list current variables\n" + " bulker envvars -a MY_VAR # add a variable\n" + " bulker envvars -r MY_VAR # remove a variable", + } + parser = _VersionInHelpParser( description=banner, epilog=additional_description) @@ -125,9 +155,12 @@ def build_argparser(): subparsers = parser.add_subparsers(dest="command") def add_subparser(cmd, description): + examples = subparser_examples.get(cmd) + epilog = "examples:\n" + examples if examples else None return subparsers.add_parser( - cmd, description=description, help=description) - + cmd, description=description, help=description, + epilog=epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) # Add subparsers sps = {} @@ -147,16 +180,14 @@ def add_subparser(cmd, description): for cmd in ["run", "activate", "load", "unload"]: sps[cmd].add_argument( "crate_registry_paths", metavar="crate-registry-paths", type=str, - help="One or more comma-separated registry path strings" - " that identify crates (e.g. bulker/demo:1.F0.0)") + help="Crate to use, e.g. bulker/demo or namespace/crate:tag") # optional for inspect and cwl2man for cmd in ["inspect"]: sps[cmd].add_argument( "crate_registry_paths", metavar="crate-registry-paths", type=str, nargs="?", default=os.getenv("BULKERCRATE", ""), - help="One or more comma-separated registry path strings" - " that identify crates (e.g. bulker/demo:1.0.0)") + help="Crate to inspect (defaults to active crate from BULKERCRATE)") for cmd in ["run", "activate"]: sps[cmd].add_argument( @@ -1282,7 +1313,7 @@ def main(): if args.command == "inspect": if args.crate_registry_paths == "": - _LOGGER.error("No active create. Inspect requires a provided crate, or a currently active create.") + _LOGGER.error("No crate specified and no active crate (BULKERCRATE not set). Run 'bulker activate ' first, or specify: bulker inspect ") sys.exit(1) manifest, cratevars = load_remote_registry_path(bulker_config, args.crate_registry_paths, @@ -1346,10 +1377,10 @@ def main(): bulker_activate(bulker_config, cratelist, echo=args.echo, strict=args.strict, prompt=args.no_prompt) except KeyError as e: parser.print_help(sys.stderr) - _LOGGER.error("{} is not an available crate".format(e)) + _LOGGER.error("{} is not an available crate. Run 'bulker list' to see loaded crates.".format(e)) sys.exit(1) except MissingCrateError as e: - _LOGGER.error("Missing crate: {}".format(e)) + _LOGGER.error("Missing crate: {}. Run 'bulker list' to see loaded crates, or 'bulker load' to add one.".format(e)) sys.exit(1) except AttributeError as e: _LOGGER.error("Your bulker config file is outdated, you need to re-initialize it: {}".format(e)) @@ -1362,10 +1393,10 @@ def main(): bulker_run(bulker_config, cratelist, args.cmd, strict=args.strict) except KeyError as e: parser.print_help(sys.stderr) - _LOGGER.error("{} is not an available crate".format(e)) + _LOGGER.error("{} is not an available crate. Run 'bulker list' to see loaded crates.".format(e)) sys.exit(1) except MissingCrateError as e: - _LOGGER.error("Missing crate: {}".format(e)) + _LOGGER.error("Missing crate: {}. Run 'bulker list' to see loaded crates, or 'bulker load' to add one.".format(e)) sys.exit(1) if args.command == "load": From 4d60447c650708eec95352f8d086cd54bb698ab6 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 17 Feb 2026 09:18:12 -0500 Subject: [PATCH 12/17] Migrate to yacman v1 API --- bulker/bulker.py | 68 +++++++++++++++---------------- requirements/requirements-all.txt | 2 +- tests/test_bulker.py | 10 ++--- 3 files changed, 38 insertions(+), 42 deletions(-) diff --git a/bulker/bulker.py b/bulker/bulker.py index a4d65e9..4849a7d 100644 --- a/bulker/bulker.py +++ b/bulker/bulker.py @@ -10,6 +10,7 @@ import psutil import sys import yacman +from yacman import write_lock import signal import shutil @@ -308,7 +309,7 @@ def bulker_envvars_add(bulker_config, variable): _LOGGER.error("You must specify a variable.") return - with bulker_config as bcfg: + with write_lock(bulker_config) as bcfg: if variable in bcfg["bulker"]["envvars"]: _LOGGER.info("Variable '{}' already present".format(variable)) else: @@ -328,8 +329,7 @@ def bulker_envvars_remove(bulker_config, variable): _LOGGER.error("You must specify a variable.") return - with bulker_config as bcfg: - bcfg.make_writable() + with write_lock(bulker_config) as bcfg: if variable in bcfg["bulker"]["envvars"]: _LOGGER.info("Removing variable '{}'".format(variable)) bcfg["bulker"]["envvars"].remove(variable) @@ -371,7 +371,7 @@ def bulker_init(config_path, template_config_path, container_engine=None): # templates_subdir = TEMPLATE_SUBDIR copytree(os.path.dirname(template_config_path), dest_templates_dir, dirs_exist_ok=True) new_template = os.path.join(dest_folder, os.path.basename(template_config_path)) - bulker_config = yacman.YAMLConfigManager(filepath=template_config_path, locked=False, skip_read_lock=True) + bulker_config = yacman.YAMLConfigManager.from_yaml_file(template_config_path) _LOGGER.debug("Engine used: {}".format(container_engine)) bulker_config["bulker"]["container_engine"] = container_engine if bulker_config["bulker"]["container_engine"] == "docker": @@ -483,9 +483,9 @@ def bulker_load(manifest, cratevars, bcfg, exe_jinja2_template, if not bcfg["bulker"]["crates"]: bcfg["bulker"]["crates"] = {} if not cratevars['namespace'] in bcfg["bulker"]["crates"]: - bcfg["bulker"]["crates"][cratevars['namespace']] = yacman.YAMLConfigManager({}) + bcfg["bulker"]["crates"][cratevars['namespace']] = {} if not cratevars['crate'] in bcfg["bulker"]["crates"][cratevars['namespace']]: - bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']] = yacman.YAMLConfigManager({}) + bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']] = {} if cratevars['tag'] in bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']]: _LOGGER.debug(bcfg["bulker"]["crates"][cratevars['namespace']][cratevars['crate']]) if not (force or query_yes_no("That manifest has already been loaded. Overwrite?")): @@ -555,7 +555,7 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): for pkg in manifest["manifest"]["commands"]: _LOGGER.debug(pkg) pkg.update(bcfg["bulker"]) # Add terms from the bulker config - pkg = copy.deepcopy(yacman.YAMLConfigManager(pkg)) # (otherwise it's just a dict) + pkg = copy.deepcopy(pkg) # We have to deepcopy it so that changes we make to pkg aren't reflected in bcfg. if pkg["container_engine"] == "singularity" and "singularity_image_folder" in pkg: @@ -678,8 +678,8 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): _LOGGER.info("Host commands available: {}".format(", ".join(host_cmdlist))) - with bcfg as bcfg: - bcfg.write() + with write_lock(bcfg) as locked_bcfg: + locked_bcfg.write() def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=True): """ @@ -1037,9 +1037,9 @@ def load_remote_registry_path(bulker_config, registry_path, filepath=None): raise Exception("No remote manifest found") data = response.read() # a `bytes` object text = data.decode('utf-8') - manifest_lines = yacman.YAMLConfigManager(yamldata=text) + manifest_lines = yacman.YAMLConfigManager.from_yaml_data(text) else: - manifest_lines = yacman.YAMLConfigManager(filepath=filepath) + manifest_lines = yacman.YAMLConfigManager.from_yaml_file(filepath) return manifest_lines, cratevars @@ -1134,13 +1134,13 @@ def parse_cwl(cwl_file): """ :param str cwl_file: CWL tool description file. """ - yam = yacman.YAMLConfigManager(filepath=cwl_file) + yam = yacman.YAMLConfigManager.from_yaml_file(cwl_file) if yam["class"] != "CommandLineTool": _LOGGER.info("CWL file of wrong class: {} ({})".format(cwl_file, yam["class"])) return None try: - maybe_base_command = yam.baseCommand - except AttributeError: + maybe_base_command = yam["baseCommand"] + except KeyError: _LOGGER.info("Can't find base command from {}".format(cwl_file)) raise BaseCommandNotFoundException(cwl_file) @@ -1184,7 +1184,7 @@ def parse_cwl(cwl_file): if str(image).startswith("$include"): print(str(image)) - x = yacman.YAMLConfigManager(yamldata=str(image)) + x = yacman.YAMLConfigManager.from_yaml_data(str(image)) file_path = str(x["$include"]) with open(os.path.join(os.path.dirname(cwl_file), file_path), 'r') as f: contents = f.read() @@ -1218,20 +1218,17 @@ def bulker_unload(bulker_config, crate_registry_paths): crate=crate, tag=tag) _LOGGER.info("Removing crate: '{}'".format(regpath)) - bulker_config.make_writable() - # bulker_config["bulker"].crates[namespace][crate][tag] = None crate_path = bulker_config["bulker"]["crates"][namespace][crate][tag] - del bulker_config["bulker"]["crates"][namespace][crate][tag] + with write_lock(bulker_config) as locked_cfg: + del locked_cfg["bulker"]["crates"][namespace][crate][tag] + if len(locked_cfg["bulker"]["crates"][namespace][crate]) == 0: + _LOGGER.info("Last tag!") + del locked_cfg["bulker"]["crates"][namespace][crate] + locked_cfg.write() try: shutil.rmtree(crate_path) except: _LOGGER.error("Error removing crate at {}. Did your crate path change? Remove it manually.".format(crate_path)) - - if len(bulker_config["bulker"]["crates"][namespace][crate]) ==0: - _LOGGER.info("Last tag!") - del bulker_config["bulker"]["crates"][namespace][crate] - with bulker_config as bcfg: - bcfg.write() removed_crates.append(regpath) if len(removed_crates) > 0: @@ -1265,9 +1262,7 @@ def main(): sys.exit(0) if args.command == "cwl2man": - bm = yacman.YAMLConfigManager() - bm.manifest = yacman.YAMLConfigManager() - bm.manifest["commands"] = [] + bm = {"manifest": {"commands": []}} baseCommandsNotFound = [] imagesNotFound = [] @@ -1275,27 +1270,28 @@ def main(): try: cmd = parse_cwl(cwl_file) if cmd: - bm.manifest["commands"].append(cmd) + bm["manifest"]["commands"].append(cmd) except BaseCommandNotFoundException as e: baseCommandsNotFound.append(e.file) except ImageNotFoundException as e: imagesNotFound.append(e.file) - _LOGGER.info("Commands added: {}".format(len(bm.manifest["commands"]))) + _LOGGER.info("Commands added: {}".format(len(bm["manifest"]["commands"]))) if len(baseCommandsNotFound) > 0: _LOGGER.info("Command not found ({}): {}".format( len(baseCommandsNotFound), baseCommandsNotFound)) if len(imagesNotFound) > 0: _LOGGER.info("Image not found ({}): {}".format( len(imagesNotFound), imagesNotFound)) - - bm.write_copy(args.manifest) + + bm_cfg = yacman.YAMLConfigManager.from_obj(bm) + bm_cfg.write_copy(args.manifest) sys.exit(0) # Any remaining commands require config so we process it now. bulkercfg = select_bulker_config(args.config) - bulker_config = yacman.YAMLConfigManager(filepath=bulkercfg, locked=False) + bulker_config = yacman.YAMLConfigManager.from_yaml_file(bulkercfg) # _LOGGER.info("Bulker config: {}".format(bulkercfg)) if args.command == "envvars": @@ -1307,7 +1303,7 @@ def main(): _LOGGER.debug("Removing env var") _is_writable(os.path.dirname(bulkercfg), check_exist=False) bulker_envvars_remove(bulker_config, args.remove) - _LOGGER.info("Envvars list: {}".format(bulker_config["bulker"].envvars)) + _LOGGER.info("Envvars list: {}".format(bulker_config["bulker"]["envvars"])) sys.exit(0) @@ -1356,7 +1352,7 @@ def main(): _LOGGER.info("Available crates:") fmt = "{namespace}/{crate}:{tag} -- {path}" - if bulker_config["bulker"].crates: + if bulker_config["bulker"]["crates"]: for namespace, crates in bulker_config["bulker"]["crates"].items(): for crate, tags in crates.items(): for tag, path in tags.items(): @@ -1400,7 +1396,7 @@ def main(): sys.exit(1) if args.command == "load": - with bulker_config as _: + with write_lock(bulker_config) as _: manifest, cratevars, exe_template_jinja, shell_template_jinja, build_template_jinja = prep_load( bulker_config, args.crate_registry_paths, args.manifest, args.build) @@ -1419,7 +1415,7 @@ def main(): if args.command == "reload": _LOGGER.info("Reloading all manifests") - with bulker_config as _: + with write_lock(bulker_config) as _: bulker_reload(bulker_config) if args.command == "unload": diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 1333fc8..c8b86bc 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -1,4 +1,4 @@ -yacman>=0.8.4 +yacman>=0.9.5 pyyaml>=5.1 logmuse>=0.2.0 jinja2 diff --git a/tests/test_bulker.py b/tests/test_bulker.py index 885569b..00f1a8c 100644 --- a/tests/test_bulker.py +++ b/tests/test_bulker.py @@ -16,7 +16,7 @@ def test_yacman(): - bc = yacman.YAMLConfigManager(filepath=DEFAULT_CONFIG_FILEPATH) + bc = yacman.YAMLConfigManager.from_yaml_file(DEFAULT_CONFIG_FILEPATH) bc bc["bulker"]["default_crate_folder"] @@ -28,7 +28,7 @@ def test_yacman(): def test_float_idx(): - data = yacman.YAMLConfigManager(yamldata=yaml_str) + data = yacman.YAMLConfigManager.from_yaml_data(yaml_str) # We should be able to access this by string, not by int index. assert(data['2'] == "two") with pytest.raises(KeyError): @@ -51,7 +51,7 @@ def test_bulker_init(): pass bulker_init(DUMMY_CFG_FILEPATH, DEFAULT_CONFIG_FILEPATH, "docker") - bulker_config = yacman.YAMLConfigManager(filepath=DEFAULT_CONFIG_FILEPATH) + bulker_config = yacman.YAMLConfigManager.from_yaml_file(DEFAULT_CONFIG_FILEPATH) manifest, cratevars = load_remote_registry_path(bulker_config, "demo", @@ -66,7 +66,7 @@ def test_bulker_init(): def test_bulker_activate(): - bulker_config = yacman.YAMLConfigManager(filepath=DEFAULT_CONFIG_FILEPATH) + bulker_config = yacman.YAMLConfigManager.from_yaml_file(DEFAULT_CONFIG_FILEPATH) def test_nonconfig_load(): @@ -85,7 +85,7 @@ def test_nonconfig_load(): pass bulker_init(DUMMY_CFG_FILEPATH, DEFAULT_CONFIG_FILEPATH, "docker") - bulker_config = yacman.YAMLConfigManager(filepath=DUMMY_CFG_FILEPATH) + bulker_config = yacman.YAMLConfigManager.from_yaml_file(DUMMY_CFG_FILEPATH) # The 'load' command will write the new crate to the config file; # we don't want it to update the template config file, so make a dummy From 7b38acab472aaa2596300e91b7ce5fb08083d078 Mon Sep 17 00:00:00 2001 From: nsheff Date: Tue, 17 Feb 2026 09:55:29 -0500 Subject: [PATCH 13/17] fix double-lock --- bulker/bulker.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bulker/bulker.py b/bulker/bulker.py index 4849a7d..c614a2e 100644 --- a/bulker/bulker.py +++ b/bulker/bulker.py @@ -678,8 +678,7 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): _LOGGER.info("Host commands available: {}".format(", ".join(host_cmdlist))) - with write_lock(bcfg) as locked_bcfg: - locked_bcfg.write() + bcfg.write() def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=True): """ From b397424d0171032be806f3870d2afa723f2e12a5 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Feb 2026 07:09:08 -0500 Subject: [PATCH 14/17] update to latest ubiquerg --- bulker/bulker.py | 3 ++- requirements/requirements-all.txt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bulker/bulker.py b/bulker/bulker.py index c614a2e..16a9282 100644 --- a/bulker/bulker.py +++ b/bulker/bulker.py @@ -678,7 +678,8 @@ def host_tool_specific_args(bcfg, pkg, hosttool_arg_key): _LOGGER.info("Host commands available: {}".format(", ".join(host_cmdlist))) - bcfg.write() + with write_lock(bcfg) as locked_cfg: + locked_cfg.write() def bulker_activate(bulker_config, cratelist, echo=False, strict=False, prompt=True): """ diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index c8b86bc..559b75e 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -2,5 +2,5 @@ yacman>=0.9.5 pyyaml>=5.1 logmuse>=0.2.0 jinja2 -ubiquerg>=0.5.1 +ubiquerg>=0.9.0 psutil From cd7a9496e0cd98c61f2e7d05c8419679db0334cf Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Feb 2026 07:09:54 -0500 Subject: [PATCH 15/17] version bump --- bulker/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bulker/_version.py b/bulker/_version.py index b4e087d..777f190 100644 --- a/bulker/_version.py +++ b/bulker/_version.py @@ -1 +1 @@ -__version__ = "0.8.0-dev" +__version__ = "0.8.0" From 157f28cfe7e36a839a71f6321f4fa1ee021bab55 Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Feb 2026 07:11:38 -0500 Subject: [PATCH 16/17] changelog --- docs/changelog.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index d01fb26..df0a2f6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,9 +1,11 @@ # Changelog -## [0.8.0] -- Unreleased -- Some routine maintenance -- Prepped for yacman v1, dropping attmap support +## [0.8.0] -- 2026-02-25 +- Migrated to yacman v1 API (`YAMLConfigManager.from_yaml_file()`, `write_lock` context managers) +- Dropped attmap dependency +- Bumped ubiquerg requirement to >=0.9.0 - Added support for Python 3.12 +- Fixed compatibility with distutils removal in Python 3.12 ## [0.7.3] -- 2021-12-08 - Fixed a bug that prevented use when installed in non-writable directory From 3943df970d38a2fe225ddae2ed5b2f89df207e3c Mon Sep 17 00:00:00 2001 From: nsheff Date: Wed, 25 Feb 2026 07:13:54 -0500 Subject: [PATCH 17/17] update python version compatibility --- .github/workflows/run-pytest.yml | 6 +++--- setup.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run-pytest.yml b/.github/workflows/run-pytest.yml index aa898ce..2114323 100644 --- a/.github/workflows/run-pytest.yml +++ b/.github/workflows/run-pytest.yml @@ -11,14 +11,14 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ["3.8", "3.12"] + python-version: ["3.10", "3.14"] os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/setup.py b/setup.py index 3aa5cb2..c55087b 100644 --- a/setup.py +++ b/setup.py @@ -32,11 +32,10 @@ description="Manager of portable multi-container computing environments", long_description=long_description, long_description_content_type='text/markdown', + python_requires=">=3.10", classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12",