From 68a727a3cb4401a113ad425616662786b0f907be Mon Sep 17 00:00:00 2001 From: landmanbester Date: Wed, 22 May 2024 14:06:01 +0200 Subject: [PATCH 01/21] change pfb-clean to pfb-imaging --- .../genesis/{pfb-clean => pfb-imaging}/latest/cgopts.yml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/clean.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/degrid.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/dist.yml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/fastim.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/forward.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/fwdbwd.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/grid.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/gridding.yml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/init.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/init_ims.yaml | 0 .../{pfb-clean => pfb-imaging}/latest/model2comps.yaml | 0 cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/out.yml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/pdopts.yml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/pmopts.yml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/restore.yaml | 0 .../genesis/{pfb-clean => pfb-imaging}/latest/spotless.yaml | 0 .../{pfb-clean => pfb-imaging}/latest/uncabbedcabs.yml | 0 cultcargo/genesis/{pfb-clean => pfb-imaging}/sync_config.py | 6 +++--- cultcargo/images/{pfb-clean => pfb-imaging}/Dockerfile | 0 cultcargo/{pfb-clean.yml => pfb-imaging.yml} | 0 21 files changed, 3 insertions(+), 3 deletions(-) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/cgopts.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/clean.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/degrid.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/dist.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/fastim.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/forward.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/fwdbwd.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/grid.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/gridding.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/init.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/init_ims.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/model2comps.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/out.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/pdopts.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/pmopts.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/restore.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/spotless.yaml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/latest/uncabbedcabs.yml (100%) rename cultcargo/genesis/{pfb-clean => pfb-imaging}/sync_config.py (82%) rename cultcargo/images/{pfb-clean => pfb-imaging}/Dockerfile (100%) rename cultcargo/{pfb-clean.yml => pfb-imaging.yml} (100%) diff --git a/cultcargo/genesis/pfb-clean/latest/cgopts.yml b/cultcargo/genesis/pfb-imaging/latest/cgopts.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/cgopts.yml rename to cultcargo/genesis/pfb-imaging/latest/cgopts.yml diff --git a/cultcargo/genesis/pfb-clean/latest/clean.yaml b/cultcargo/genesis/pfb-imaging/latest/clean.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/clean.yaml rename to cultcargo/genesis/pfb-imaging/latest/clean.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/degrid.yaml b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/degrid.yaml rename to cultcargo/genesis/pfb-imaging/latest/degrid.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/dist.yml b/cultcargo/genesis/pfb-imaging/latest/dist.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/dist.yml rename to cultcargo/genesis/pfb-imaging/latest/dist.yml diff --git a/cultcargo/genesis/pfb-clean/latest/fastim.yaml b/cultcargo/genesis/pfb-imaging/latest/fastim.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/fastim.yaml rename to cultcargo/genesis/pfb-imaging/latest/fastim.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/forward.yaml b/cultcargo/genesis/pfb-imaging/latest/forward.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/forward.yaml rename to cultcargo/genesis/pfb-imaging/latest/forward.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/fwdbwd.yaml b/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/fwdbwd.yaml rename to cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/grid.yaml b/cultcargo/genesis/pfb-imaging/latest/grid.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/grid.yaml rename to cultcargo/genesis/pfb-imaging/latest/grid.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/gridding.yml b/cultcargo/genesis/pfb-imaging/latest/gridding.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/gridding.yml rename to cultcargo/genesis/pfb-imaging/latest/gridding.yml diff --git a/cultcargo/genesis/pfb-clean/latest/init.yaml b/cultcargo/genesis/pfb-imaging/latest/init.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/init.yaml rename to cultcargo/genesis/pfb-imaging/latest/init.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/init_ims.yaml b/cultcargo/genesis/pfb-imaging/latest/init_ims.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/init_ims.yaml rename to cultcargo/genesis/pfb-imaging/latest/init_ims.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/model2comps.yaml b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/model2comps.yaml rename to cultcargo/genesis/pfb-imaging/latest/model2comps.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/out.yml b/cultcargo/genesis/pfb-imaging/latest/out.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/out.yml rename to cultcargo/genesis/pfb-imaging/latest/out.yml diff --git a/cultcargo/genesis/pfb-clean/latest/pdopts.yml b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/pdopts.yml rename to cultcargo/genesis/pfb-imaging/latest/pdopts.yml diff --git a/cultcargo/genesis/pfb-clean/latest/pmopts.yml b/cultcargo/genesis/pfb-imaging/latest/pmopts.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/pmopts.yml rename to cultcargo/genesis/pfb-imaging/latest/pmopts.yml diff --git a/cultcargo/genesis/pfb-clean/latest/restore.yaml b/cultcargo/genesis/pfb-imaging/latest/restore.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/restore.yaml rename to cultcargo/genesis/pfb-imaging/latest/restore.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/spotless.yaml rename to cultcargo/genesis/pfb-imaging/latest/spotless.yaml diff --git a/cultcargo/genesis/pfb-clean/latest/uncabbedcabs.yml b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/uncabbedcabs.yml rename to cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml diff --git a/cultcargo/genesis/pfb-clean/sync_config.py b/cultcargo/genesis/pfb-imaging/sync_config.py similarity index 82% rename from cultcargo/genesis/pfb-clean/sync_config.py rename to cultcargo/genesis/pfb-imaging/sync_config.py index c94d8d1..25adc2f 100644 --- a/cultcargo/genesis/pfb-clean/sync_config.py +++ b/cultcargo/genesis/pfb-imaging/sync_config.py @@ -5,13 +5,13 @@ yaml.indent(mapping=2, sequence=4, offset=2) import cultcargo -mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-clean/latest' +mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-imaging/latest' # import ipdb; ipdb.set_trace() -content_path = 'https://raw.githubusercontent.com/ratt-ru/pfb-clean/awskube/pfb/parser' +content_path = 'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/band_actors/pfb/parser' # returns list of files in subfolder -response = requests.get('https://api.github.com/repos/ratt-ru/pfb-clean/contents/pfb/parser?ref=awskube') +response = requests.get('https://api.github.com/repos/ratt-ru/pfb-imaging/contents/pfb/parser?ref=band_actors') for r in response.json(): name = r['name'] print(name) diff --git a/cultcargo/images/pfb-clean/Dockerfile b/cultcargo/images/pfb-imaging/Dockerfile similarity index 100% rename from cultcargo/images/pfb-clean/Dockerfile rename to cultcargo/images/pfb-imaging/Dockerfile diff --git a/cultcargo/pfb-clean.yml b/cultcargo/pfb-imaging.yml similarity index 100% rename from cultcargo/pfb-clean.yml rename to cultcargo/pfb-imaging.yml From e3cdc3aedd399a6aacd5ed3ca75e24358886f2cf Mon Sep 17 00:00:00 2001 From: landmanbester Date: Wed, 22 May 2024 15:27:31 +0200 Subject: [PATCH 02/21] add updated pfb-imaging parsets --- .../genesis/pfb-imaging/latest/degrid.yaml | 2 +- .../genesis/pfb-imaging/latest/fastim.yaml | 108 ++++++++++---- .../genesis/pfb-imaging/latest/fluxmop.yaml | 56 ++++++++ .../genesis/pfb-imaging/latest/fwdbwd.yaml | 2 +- .../genesis/pfb-imaging/latest/grid.yaml | 20 +-- .../genesis/pfb-imaging/latest/imit.yaml | 97 +++++++++++++ .../genesis/pfb-imaging/latest/init.yaml | 8 +- .../genesis/pfb-imaging/latest/klean.yaml | 124 ++++++++++++++++ .../pfb-imaging/latest/model2comps.yaml | 17 ++- .../genesis/pfb-imaging/latest/pdopts.yml | 6 +- .../genesis/pfb-imaging/latest/restore.yaml | 13 +- .../genesis/pfb-imaging/latest/smoovie.yaml | 132 ++++++++++++++++++ .../genesis/pfb-imaging/latest/spotless.yaml | 89 +++++++++++- .../pfb-imaging/latest/uncabbedcabs.yml | 21 ++- 14 files changed, 631 insertions(+), 64 deletions(-) create mode 100644 cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml create mode 100644 cultcargo/genesis/pfb-imaging/latest/imit.yaml create mode 100644 cultcargo/genesis/pfb-imaging/latest/klean.yaml create mode 100644 cultcargo/genesis/pfb-imaging/latest/smoovie.yaml diff --git a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml index 1023ddb..50c2dd6 100644 --- a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml @@ -1,6 +1,6 @@ inputs: ms: - dtype: URI + dtype: List[URI] required: true abbreviation: ms info: Path to measurement set. diff --git a/cultcargo/genesis/pfb-imaging/latest/fastim.yaml b/cultcargo/genesis/pfb-imaging/latest/fastim.yaml index 6248b60..deb20b9 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fastim.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fastim.yaml @@ -1,6 +1,6 @@ inputs: ms: - dtype: List[str] + dtype: URI required: true abbreviation: ms info: Path to measurement set @@ -24,9 +24,10 @@ inputs: dtype: bool default: false info: Allow overwrite of output xds - radec: - dtype: Optional[List[any]] - info: Rephase all images to this radec specified in radians + transfer-model-from: + dtype: str + abbreviation: tmf + info: Name of dataset to use for model initialisation data-column: dtype: str default: DATA @@ -47,24 +48,26 @@ inputs: abbreviation: fc info: Column containing data flags. Must be the same across MSs gain-table: - dtype: List[str] - info: Path to Quartical gain table containing NET gains. There must be a table - for each MS and glob(ms) and glob(gt) should match up - gain-term: - dtype: str - default: NET - info: Which gain term to use. Default is NET + dtype: URI + info: Path to Quartical gain table containing NET gains. integrations-per-image: dtype: int abbreviation: ipi default: -1 info: Number of time integrations per image. Default (-1, 0, None) -> dataset per scan. - channels-per-image: + channels-per-degrid-image: + dtype: int + abbreviation: cpdi + default: 32 + info: Number of channels per image for degridding resolution. Any of (-1, 0, None) + implies single dataset per spw. + channels-per-grid-image: dtype: int - abbreviation: cpi + abbreviation: cpgi default: -1 - info: Number of channels per image. Default (-1, 0, None) -> dataset per spw. + info: Number of channels per image for gridding resolution. Default of (-1, 0, + None) implies a single dataset per spw. precision: dtype: str default: double @@ -72,25 +75,80 @@ inputs: - single - double info: Gridding precision - bda-decorr: - dtype: float - default: 1.0 - info: BDA decorrelation factor. Only has an effect if less than one - beam-model: - dtype: str - info: Beam model to use - max-field-of-view: + field-of-view: dtype: float abbreviation: fov default: 3.0 info: Field of view in degrees - chan-average: + super-resolution-factor: + dtype: float + default: 2 + abbreviation: srf + info: Will over-sample Nyquist by this factor at max frequency + cell-size: + dtype: float + info: Cell size in arc-seconds + nx: + dtype: int + info: Number of x pixels + ny: + dtype: int + info: Number of y pixels + robustness: + dtype: float + abbreviation: rob + info: Robustness factor for Briggs weighting. None means natural + filter-extreme-counts: + dtype: bool + default: false + info: Replace extreme outliers in the weighting grid by their local mean + filter-nbox: + dtype: int + default: 16 + info: The size of the box to compute local mean over + filter-level: + dtype: float + default: 10 + info: The level above local mean that renders a grid weight extreme + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + l2reweight-dof: + dtype: float + info: The degrees of freedom parameter for L2 reweighting. The default (None) + means no reweighting. A sensible value for this parameter depends on the level + of RFI in the data. Small values (eg. 2) result in aggressive reweighting and + should be avoided if the model is still incomplete. + host-address: + dtype: str + abbreviation: ha + info: Address where the distributed client lives. Uses LocalCluster if no address + is provided and scheduler is set to distributed. + nworkers: + dtype: int + default: 1 + abbreviation: nw + info: Number of worker processes. Use with distributed scheduler. + nthreads-dask: dtype: int default: 1 - info: Average this number if channels together + abbreviation: ntd + info: Number of dask threads. This will largely determine the memory footprint. + Used as nthreads per worker when using distributed scheduler. + nvthreads: + dtype: int + abbreviation: nvt + info: Number of threads used to scale vertically (eg. for FFTs and gridding). + Each dask thread can in principle spawn this many threads. Will attempt to use + half the available threads by default. + progressbar: + dtype: bool + default: true + info: Display progress. Use --no-progressbar to deactivate. _include: - - (.)dist.yml + - (.)gridding.yml - (.)out.yml outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml new file mode 100644 index 0000000..dc1a0bc --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml @@ -0,0 +1,56 @@ +inputs: + nband: + dtype: int + required: true + abbreviation: nb + info: Number of imaging bands + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data product + mask: + dtype: str + abbreviation: mask + info: Either path to mask.fits or set to model to construct from model + zero-model-outside-mask: + dtype: bool + default: true + info: Make sure the input model is zero outside the mask. Only has an effect if + an external mask has been passed in. A major cycle will be triggered to recompute + the residual after zeroing. + min-model: + dtype: float + default: 1e-5 + info: If using mask to construct model construct it where model > min-model + sigmainv: + dtype: float + default: 1e-5 + abbreviation: sinv + info: Standard deviation of assumed GRF prior + model-name: + dtype: str + default: MODEL + info: Name of the model to update + gamma: + dtype: float + default: 0.99 + info: Step size of update + use-psf: + dtype: bool + default: true + info: Whether to approximate the Hessian as a convolution by the PSF + fits-mfs: + dtype: bool + default: true + info: Output MFS fits files + fits-cubes: + dtype: bool + default: false + info: Output fits cubes + _include: + - (.)gridding.yml + - (.)cgopts.yml + - (.)dist.yml + - (.)out.yml + +outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml b/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml index e1c54af..18a5a72 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml @@ -21,7 +21,7 @@ inputs: required: true abbreviation: nb info: Number of imaging bands - postfix: + suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products diff --git a/cultcargo/genesis/pfb-imaging/latest/grid.yaml b/cultcargo/genesis/pfb-imaging/latest/grid.yaml index 7a4db83..ed2dea6 100644 --- a/cultcargo/genesis/pfb-imaging/latest/grid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/grid.yaml @@ -1,13 +1,12 @@ inputs: xds: - info: Optional explicit path to xds. Set using output-filename and postfix by - default. + info: Optional explicit path to xds. Set using output-filename and suffix by default. nband: dtype: int required: true abbreviation: nb info: Number of imaging bands - postfix: + suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products @@ -18,12 +17,17 @@ inputs: overwrite: dtype: bool default: false - info: Allow overwriting of image space data products. Specify postfix to create + info: Allow overwriting of image space data products. Specify suffix to create a new data set. transfer-model-from: dtype: str abbreviation: tmf info: Name of dataset to use for model initialisation + use-best-model: + dtype: bool + default: false + info: If this flag is set MODEL_BEST will be used as the model unless transfer-model-from + is specified. By default MODEL will be used as the model. robustness: dtype: float abbreviation: rob @@ -53,10 +57,6 @@ inputs: dtype: bool default: true info: Compute effectve image space weights - residual: - dtype: bool - default: false - info: Compute the residual image field-of-view: dtype: float abbreviation: fov @@ -78,7 +78,7 @@ inputs: info: Number of y pixels filter-extreme-counts: dtype: bool - default: false + default: true info: Replace extreme outliers in the weighting grid by their local mean filter-nbox: dtype: int @@ -86,7 +86,7 @@ inputs: info: The size of the box to compute local mean over filter-level: dtype: float - default: 10 + default: 5 info: The level above local mean that renders a grid weight extreme target: dtype: str diff --git a/cultcargo/genesis/pfb-imaging/latest/imit.yaml b/cultcargo/genesis/pfb-imaging/latest/imit.yaml new file mode 100644 index 0000000..d7c8a3d --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/imit.yaml @@ -0,0 +1,97 @@ +inputs: + ms: + dtype: List[URI] + required: true + abbreviation: ms + info: Path to measurement set + scans: + dtype: List[int] + info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated + list 0,2 if running from CLI + ddids: + dtype: List[int] + info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated + list 0,2 if running from CLI + fields: + dtype: List[int] + info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list + 0,2 if running from CLI + freq-range: + dtype: str + info: Frequency range to image in Hz. Specify as a string with colon delimiter + eg. '1e9:1.1e9' + overwrite: + dtype: bool + default: false + info: Allow overwrite of output xds + radec: + dtype: Optional[List[any]] + info: Rephase all images to this radec specified in radians + data-column: + dtype: str + default: DATA + abbreviation: dc + info: Data column to image. Must be the same across MSs + weight-column: + dtype: str + abbreviation: wc + info: Column containing natural weights. Must be the same across MSs + sigma-column: + dtype: str + abbreviation: sc + info: Column containing standard devations. Will be used to initialise natural + weights if detected. Must be the same across MSs + flag-column: + dtype: str + default: FLAG + abbreviation: fc + info: Column containing data flags. Must be the same across MSs + gain-table: + dtype: List[str] + info: Path to Quartical gain table containing NET gains. There must be a table + for each MS and glob(ms) and glob(gt) should match up when running from CLI. + integrations-per-image: + dtype: int + abbreviation: ipi + default: -1 + info: Number of time integrations per image. Default (-1, 0, None) -> dataset + per scan. + channels-per-image: + dtype: int + abbreviation: cpi + default: 32 + info: Number of channels per image for degridding resolution. Any of (-1, 0, None) + implies single dataset per spw. + precision: + dtype: str + default: double + choices: + - single + - double + info: Gridding precision + bda-decorr: + dtype: float + default: 1.0 + info: BDA decorrelation factor. Only has an effect if less than one + beam-model: + dtype: str + info: Beam model to use + chan-average: + dtype: int + default: 1 + info: Average this number if channels together + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + progressbar: + dtype: bool + default: true + info: Display progress. Use --no-progressbar to deactivate. + + + _include: + - (.)dist.yml + - (.)out.yml + +outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/init.yaml b/cultcargo/genesis/pfb-imaging/latest/init.yaml index 7d6177a..5639c1f 100644 --- a/cultcargo/genesis/pfb-imaging/latest/init.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/init.yaml @@ -50,10 +50,6 @@ inputs: dtype: List[str] info: Path to Quartical gain table containing NET gains. There must be a table for each MS and glob(ms) and glob(gt) should match up - gain-term: - dtype: str - default: NET - info: Which gain term to use. Default is NET integrations-per-image: dtype: int abbreviation: ipi @@ -88,6 +84,10 @@ inputs: dtype: int default: 1 info: Average this number if channels together + visualize-graph: + dtype: bool + default: false + info: Visualise the compute graph. _include: - (.)dist.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/klean.yaml b/cultcargo/genesis/pfb-imaging/latest/klean.yaml new file mode 100644 index 0000000..3213b32 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/klean.yaml @@ -0,0 +1,124 @@ +inputs: + nband: + dtype: int + required: true + abbreviation: nb + info: Number of imaging bands + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products. + mask: + dtype: str + abbreviation: mask + info: Path to mask.fits + dirosion: + dtype: int + default: 1 + info: Perform dilation followed by erosion with structure element of this width + fits-mfs: + dtype: bool + default: true + info: Output MFS fits files + fits-cubes: + dtype: bool + default: false + info: Output fits cubes + mop-flux: + dtype: bool + default: true + info: Trigger PCG based flux mop if minor cycle stalls, the final threshold is + reached or on the final iteration. + mop-gamma: + dtype: float + default: 0.65 + info: Step size for flux mop. Should be between (0,1). A value of 1 is most aggressive. + niter: + dtype: int + default: 5 + abbreviation: niter + info: Number of major iterations + threshold: + dtype: float + info: Absolute threshold at which to stop cleaning. By default it is set automatically + using sigmathreshold parameter + abbreviation: th + sigmathreshold: + dtype: float + default: 3.0 + info: Multiple of the rms at which to terminate the minor cycle + gamma: + dtype: float + default: 0.1 + abbreviation: gamma + info: Minor loop gain + peak-factor: + dtype: float + default: 0.15 + abbreviation: pf + info: Peak factor + sub-peak-factor: + dtype: float + default: 0.75 + abbreviation: spf + info: Peak factor of sub-minor loop + minor-maxit: + dtype: int + default: 50 + abbreviation: mmaxit + info: Maximum number of PSF convolutions between major cycles + subminor-maxit: + dtype: int + default: 1000 + abbreviation: smmaxit + info: Maximum number of iterations for the sub-minor cycle + verbose: + dtype: int + default: 1 + abbreviation: verb + info: Verbosity level. Set to 2 for maximum verbosity, 0 for silence + report-freq: + dtype: int + default: 10 + abbreviation: rf + info: Report frequency for minor cycles + cg-tol: + dtype: float + default: 1e-2 + abbreviation: cgtol + info: Tolreance of conjugate gradient algorithm + cg-maxit: + dtype: int + default: 100 + abbreviation: cgmaxit + info: Maximum iterations for conjugate gradient algorithm + cg-minit: + dtype: int + default: 1 + abbreviation: cgminit + info: Minimum iterations for conjugate gradient algorithm + cg-verbose: + dtype: int + default: 1 + abbreviation: cgverb + info: Verbosity of conjugate gradient algorithm. Set to > 1 for debugging, 0 for + silence + cg-report_freq: + dtype: int + default: 100 + abbreviation: cgrf + info: Report frequency of conjugate gradient algorithm + backtrack: + dtype: bool + default: true + info: Ensure residual decreases at every iteration + memory-greedy: + dtype: bool + default: false + info: Holds data in memory if set + _include: + - (.)gridding.yml + - (.)dist.yml + - (.)out.yml + +outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml index aa3b872..80b8ec7 100644 --- a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml @@ -9,14 +9,16 @@ inputs: info: Order of interpolating polynomial for frequency axis nbasist: dtype: int - info: Order of interpolating polynomial for frequency axis + default: 1 + info: Order of interpolating polynomial for time axis. This is hypothetical for + the time being. fit-mode: dtype: str default: Legendre min-val: dtype: float info: Only fit components above this flux level - postfix: + suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products @@ -36,6 +38,17 @@ inputs: dtype: float default: 1e-10 info: Multiple of the identity to add to the hessian for stability + model-out: + dtype: str + info: Optional explicit output name. Otherwise the default naming convention is + used. + out-format: + dtype: str + default: zarr + choices: + - zarr + - json + info: Format to dump model to. _include: - (.)dist.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/pdopts.yml b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml index bdfad58..7e0db69 100644 --- a/cultcargo/genesis/pfb-imaging/latest/pdopts.yml +++ b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml @@ -1,11 +1,11 @@ pd-tol: dtype: float - default: 1e-3 + default: 3e-4 info: Tolreance of primal dual algorithm abbreviation: pdtol pd-maxit: dtype: int - default: 250 + default: 450 info: Maximum iterations for primal dual algorithm abbreviation: pdmaxit pd-verbose: @@ -15,6 +15,6 @@ pd-verbose: abbreviation: pdverb pd-report-freq: dtype: int - default: 10 + default: 50 info: Report frequency of primal dual algorithm abbreviation: pdrf diff --git a/cultcargo/genesis/pfb-imaging/latest/restore.yaml b/cultcargo/genesis/pfb-imaging/latest/restore.yaml index e984a8d..1e6bad2 100644 --- a/cultcargo/genesis/pfb-imaging/latest/restore.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/restore.yaml @@ -6,7 +6,7 @@ inputs: dtype: str abbreviation: mname default: MODEL - info: Name of model in mds + info: Name of model in dds residual-name: dtype: str abbreviation: rname @@ -17,18 +17,19 @@ inputs: required: true abbreviation: nb info: Number of imaging bands - postfix: + suffix: dtype: str default: main - info: Can be used to specify a custom name for the image space data \ products + info: Can be used to specify a custom name for the image space data products outputs: dtype: str default: mMrRiI - info: Output products. (m)odel, (r)esidual, (i)mage, (c)lean beam. \ Captitals - correspond to cubes. + info: Output products (m)odel, (r)esidual, (i)mage, (c)lean beam, (d)irty, (f)ft_residuals + (amplitude and phase will be produced). Use captitals to produce corresponding + cubes. overwrite: dtype: bool default: false - info: Allow overwrite of output xds + info: Allow overwriting fits files outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml new file mode 100644 index 0000000..c67ecf5 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml @@ -0,0 +1,132 @@ +inputs: + fds: + dtype: str + required: true + info: Path to fds produced by fastim worker + outname: + dtype: str + required: true + info: The name of the output. Fits images will be suffixed by band and time ID's. + Mp4 movies will suffixed by .mp4 + animate-axis: + dtype: str + default: time + choices: + - time + - freq + info: Which axis to animate + fps: + dtype: int + default: 8 + info: Frames per second + min-frac: + dtype: float + default: 3.5 + info: vmin will be set as min-frac * median(rms) + max-frac: + dtype: float + default: 3.5 + info: vmax will be set as min-frac * median(rms) + cmap: + dtype: str + default: Grays + info: Any valid matplotlib cmap + text-colour: + dtype: str + default: cyan + info: Colour to render text on image + log-directory: + dtype: str + required: true + abbreviation: ldir + info: Directory to write logs and performance reports to + freq-bin: + dtype: int + default: 1 + info: How many freqs to bin into single frame + time-bin: + dtype: int + default: 1 + info: How many times to bin into single frame + freq-range: + dtype: str + info: Frequency range to consider. Specify as a string with colon delimiter eg. + '1e9:1.1e9' + time-range: + dtype: str + info: Time range to consider in mjds Specify as a string with colon delimiter + eg. '5e7:5.5e7' + overwrite: + dtype: bool + default: false + info: Allow overwrite of output files with the same names. + fits-mfs: + dtype: bool + default: true + info: Output MFS fits files + fits-cubes: + dtype: bool + default: false + info: Output fits cubes + movie-mfs: + dtype: bool + default: true + info: Output MFS movie + movie-cubes: + dtype: bool + default: false + info: Output movie per freq band + host-address: + dtype: str + abbreviation: ha + info: Address where the distributed client lives. Uses LocalCluster if no address + is provided and scheduler is set to distributed. + nworkers: + dtype: int + default: 1 + abbreviation: nw + info: Number of worker processes. Use with distributed scheduler. + nthreads: + dtype: int + default: 1 + abbreviation: ntd + info: Number of dask threads. This will largely determine the memory footprint. + Used as nthreads per worker when using distributed scheduler. + # TBD + field-of-view: + dtype: float + abbreviation: fov + default: 3.0 + info: Field of view in degrees + nx: + dtype: int + info: Number of x pixels + ny: + dtype: int + info: Number of y pixels + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + respect-scan-boundaries: + dtype: bool + default: true + info: Whether to allow averaging across scan boundaries + out-format: + dtype: str + default: gif + choices: + - gif + - mp4 + info: The format to write movie out in. gifs are usually better quality but can + get quite large. mp4 quality not currently great with streamjoy. + optimize: + dtype: bool + default: false + info: To try and optimize the resulting gif. Only possible if gifsicle is installed. + crf: + dtype: int + default: 12 + info: Constant rate factor for controlling mp4 output quality. + +outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml index e07ba1a..4c7e59e 100644 --- a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml @@ -10,7 +10,7 @@ inputs: required: true abbreviation: nb info: Number of imaging bands - postfix: + suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products @@ -27,19 +27,25 @@ inputs: l1reweight-from: dtype: int default: 5 - info: Start doing l1reweights after this many iterations + info: L1 reweighting will kick in either at convergence or after this many iterations. + Set to a negative value to disbale L1 reweighting. hessnorm: dtype: float abbreviation: hessnorm info: Spectral norm of hessian approximation rmsfactor: dtype: float - default: 3.0 + default: 1.0 info: By default will threshold by rmsfactor*rms at every iteration gamma: dtype: float default: 1.0 info: Step size of update + alpha: + dtype: float + default: 2 + info: Controls how aggressively the l1reweighting is applied. Larger values correspond + to more agressive reweighting. positivity: dtype: int default: 1 @@ -51,12 +57,12 @@ inputs: constraint and 2 -> strong positivity i.e. all pixels in a band > 0 niter: dtype: int - default: 5 + default: 10 abbreviation: niter - info: Number of iterations. L21 reweighting will take place after every iteration + info: Number of iterations. tol: dtype: float - default: 1e-3 + default: 5e-4 info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) < tol fits-mfs: @@ -71,5 +77,76 @@ inputs: dtype: bool default: false info: Holds data in memory if set + init-factor: + dtype: float + default: 0.5 + info: Lower the initial rmsfactor by this amount + diverge-count: + dtype: int + default: 5 + info: Will terminate the algorithm if the rms increases this many times. Set to + > niter to disable this check. + field-of-view: + dtype: float + abbreviation: fov + info: Field of view in degrees + metavar: DEG + super-resolution-factor: + dtype: float + default: 2 + abbreviation: srf + info: Will over-sample Nyquist by this factor at max frequency + cell-size: + dtype: float + info: Cell size in arc-seconds + nx: + dtype: int + info: Number of x pixels + ny: + dtype: int + info: Number of y pixels + robustness: + dtype: float + default: 0.0 + abbreviation: rob + info: Robustness value for Briggs weighting + psf-oversize: + dtype: float + default: 2.0 + abbreviation: psfo + info: Size of PSF relative to dirty image + filter-counts-level: + dtype: float + default: 5.0 + info: Set minimum counts in the uniform weighting grid to the median divided by + this value. This is useful to avoid artificially up-weighting nearly empty uv-cells. + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + concat: + dtype: bool + default: true + info: Concatenate datasets by row/time. This will result in a single dirty image, + psf and beam per band. + mf-weighting: + dtype: bool + default: false + info: Base the imaging weights on the uniform weights computed over all data instead + of per band. This ensures that the MFS image can truly be uniformly weighted. + l2-reweight-dof: + dtype: float + default: 3.5 + info: The degrees of freedom parameter for performing L2 reweighting. L2 reweighting + is triggered if L1 reweighting is active and convergence is reached and will + be applied a maximum of max-l2-reweight times. + max-l2-reweight: + dtype: int + default: 5 + info: The maximum number of L2 reweighting steps. + reset-cache: + dtype: bool + default: false + info: Hard reset the cache outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml index e8aa5bb..55bc110 100644 --- a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml +++ b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml @@ -7,6 +7,15 @@ pfb.init: _include: - (.)init.yaml +pfb.imit: + command: pfb.workers.imit.imit + flavour: python + policies: + pass_missing_as_none: true + + _include: + - (.)imit.yaml + pfb.grid: command: pfb.workers.grid.grid flavour: python @@ -25,14 +34,14 @@ pfb.degrid: _include: - (.)degrid.yaml -pfb.clean: - command: pfb.workers.clean.clean +pfb.klean: + command: pfb.workers.klean.klean flavour: python policies: pass_missing_as_none: true _include: - - (.)clean.yaml + - (.)klean.yaml pfb.restore: command: pfb.workers.restore.restore @@ -52,14 +61,14 @@ pfb.fwdbwd: _include: - (.)fwdbwd.yaml -pfb.forward: - command: pfb.workers.forward.forward +pfb.fluxmop: + command: pfb.workers.fluxmop.fluxmop flavour: python policies: pass_missing_as_none: true _include: - - (.)forward.yaml + - (.)fluxmop.yaml pfb.spotless: command: pfb.workers.spotless.spotless From 00be59c6d1f71865373f6be0c9e528b360ca5b79 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Wed, 22 May 2024 16:55:50 +0200 Subject: [PATCH 03/21] remove stale pfb config files --- cultcargo/builder/cargo-manifest.yml | 4 +- .../genesis/pfb-imaging/latest/clean.yaml | 124 ------------------ .../genesis/pfb-imaging/latest/forward.yaml | 42 ------ .../genesis/pfb-imaging/latest/init_ims.yaml | 102 -------------- cultcargo/pfb-imaging.yml | 42 +++--- 5 files changed, 23 insertions(+), 291 deletions(-) delete mode 100644 cultcargo/genesis/pfb-imaging/latest/clean.yaml delete mode 100644 cultcargo/genesis/pfb-imaging/latest/forward.yaml delete mode 100644 cultcargo/genesis/pfb-imaging/latest/init_ims.yaml diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index ce82582..5be7158 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -153,10 +153,10 @@ images: master: package: git+https://github.com/ratt-ru/CubiCal - pfb-clean: + pfb-imaging: versions: latest: - package: git+https://github.com/ratt-ru/pfb-clean@awskube + package: git+https://github.com/ratt-ru/pfb-imaging@band_actors ddfacet: assign: diff --git a/cultcargo/genesis/pfb-imaging/latest/clean.yaml b/cultcargo/genesis/pfb-imaging/latest/clean.yaml deleted file mode 100644 index c8c6f61..0000000 --- a/cultcargo/genesis/pfb-imaging/latest/clean.yaml +++ /dev/null @@ -1,124 +0,0 @@ -inputs: - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data products. - mask: - dtype: str - abbreviation: mask - info: Path to mask.fits - dirosion: - dtype: int - default: 1 - info: Perform dilation followed by erosion with structure element of this width - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - mop-flux: - dtype: bool - default: true - info: Trigger PCG based flux mop if minor cycle stalls, the final threshold is - reached or on the final iteration. - mop-gamma: - dtype: float - default: 0.65 - info: Step size for flux mop. Should be between (0,1). A value of 1 is most aggressive. - nmiter: - dtype: int - default: 5 - abbreviation: nmiter - info: Number of major iterations - threshold: - dtype: float - info: Absolute threshold at which to stop cleaning. By default it is set automatically - using sigmathreshold parameter - abbreviation: th - sigmathreshold: - dtype: float - default: 2.0 - info: Multiple of the rms at which to terminate the minor cycle - gamma: - dtype: float - default: 0.1 - abbreviation: gamma - info: Minor loop gain - peak-factor: - dtype: float - default: 0.15 - abbreviation: pf - info: Peak factor - sub-peak-factor: - dtype: float - default: 0.75 - abbreviation: spf - info: Peak factor of sub-minor loop - minor-maxit: - dtype: int - default: 50 - abbreviation: mmaxit - info: Maximum number of PSF convolutions between major cycles - subminor-maxit: - dtype: int - default: 1000 - abbreviation: smmaxit - info: Maximum number of iterations for the sub-minor cycle - verbose: - dtype: int - default: 1 - abbreviation: verb - info: Verbosity level. Set to 2 for maximum verbosity, 0 for silence - report-freq: - dtype: int - default: 10 - abbreviation: rf - info: Report frequency for minor cycles - cg-tol: - dtype: float - default: 1e-2 - abbreviation: cgtol - info: Tolreance of conjugate gradient algorithm - cg-maxit: - dtype: int - default: 100 - abbreviation: cgmaxit - info: Maximum iterations for conjugate gradient algorithm - cg-minit: - dtype: int - default: 1 - abbreviation: cgminit - info: Minimum iterations for conjugate gradient algorithm - cg-verbose: - dtype: int - default: 1 - abbreviation: cgverb - info: Verbosity of conjugate gradient algorithm. Set to > 1 for debugging, 0 for - silence - cg-report_freq: - dtype: int - default: 100 - abbreviation: cgrf - info: Report frequency of conjugate gradient algorithm - backtrack: - dtype: bool - default: true - info: Ensure residual decreases at every iteration - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set - _include: - - (.)gridding.yml - - (.)dist.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/forward.yaml b/cultcargo/genesis/pfb-imaging/latest/forward.yaml deleted file mode 100644 index fbf816a..0000000 --- a/cultcargo/genesis/pfb-imaging/latest/forward.yaml +++ /dev/null @@ -1,42 +0,0 @@ -inputs: - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data product - mask: - dtype: str - abbreviation: mask - info: Either path to mask.fits or set to model to construct from model - sigmainv: - dtype: float - default: 1e-5 - abbreviation: sinv - info: Standard deviation of assumed GRF prior - gamma: - dtype: float - default: 1.0 - info: Step size of update - use-psf: - dtype: bool - default: true - info: Whether to approximate the Hessian as a convolution by the PSF - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - _include: - - (.)gridding.yml - - (.)cgopts.yml - - (.)dist.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/init_ims.yaml b/cultcargo/genesis/pfb-imaging/latest/init_ims.yaml deleted file mode 100644 index d461637..0000000 --- a/cultcargo/genesis/pfb-imaging/latest/init_ims.yaml +++ /dev/null @@ -1,102 +0,0 @@ -inputs: - ms: - dtype: str - required: true - info: 'Path to measurement set.' - metavar: MS - abbreviation: ms - output_filename: - dtype: str - required: true - info: 'Basename of output' - metavar: BASENAME - abbreviation: o - overwrite: - dtype: bool - default: false - info: Allow overwrite of output xds - nband: - dtype: int - required: true - abbreviation: nb - info: 'Number of imaging bands' - data_column: - dtype: str - default: DATA - abbreviation: dc - info: 'Data column to image. Must be the same across MSs' - metavar: COLUMN - weight_column: - dtype: str - default: WEIGHT_SPECTRUM - abbreviation: wc - info: 'Column containing natural weights. Must be the same across MSs' - metavar: COLUMN - sigma_column: - dtype: str - abbreviation: sc - info: 'Column containing standard devations. \ - Will be used to initialise natural weights if detected. \ - Must be the same across MSs' - metavar: COLUMN - flag_column: - dtype: str - default: FLAG - abbreviation: fc - info: 'Column containing data flags. Must be the same across MSs' - metavar: COLUMN - gain_table: - dtype: str - info: "Path to Quartical gain table containing NET gains.\ - There must be a table for each MS and glob(ms) and glob(gt) \ - should match up." - gain_term: - dtype: str - default: NET - info: "Which gain term to use. Default is NET" - product: - dtype: str - abbreviation: p - default: I - info: 'Imaging products to produce. Options are I, Q, U, V. \ - Only single Stokes products are currently supported' - utimes_per_chunk: - dtype: int - info: 'Unique times per row chunk' - abbreviation: utpc - default: -1 - group_by_field: - dtype: bool - default: True - info: "Split data by field" - group_by_ddid: - dtype: bool - default: True - info: "Split data by DDID" - group_by_scan: - dtype: bool - default: True - info: "Split data by scan" - precision: - dtype: str - default: double - info: '"single" or "double"' - bda_decorr: - dtype: float - default: 1.0 - info: "BDA decorrelation factor. Only has an effect if less than one" - beam_model: - dtype: str - info: "Beam model to use" - max_field_of_view: - dtype: float - abbreviation: fov - default: 3.0 - info: 'Field of view in degrees' - metavar: DEG - - _include: - - dist.yml - -outputs: - {} diff --git a/cultcargo/pfb-imaging.yml b/cultcargo/pfb-imaging.yml index 828a6c3..8dea45d 100644 --- a/cultcargo/pfb-imaging.yml +++ b/cultcargo/pfb-imaging.yml @@ -4,46 +4,46 @@ _include: vars: cult-cargo: - pfb-clean: + pfb-imaging: image: _use: vars.cult-cargo.images - name: pfb-clean + name: pfb-imaging cabs: - _include: genesis/pfb-clean/latest/uncabbedcabs.yml + _include: genesis/pfb-imaging/latest/uncabbedcabs.yml - pfb-init: + pfb.init: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-grid: + pfb.grid: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-degrid: + pfb.degrid: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-clean: + pfb.klean: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-restore: + pfb.restore: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-fwdbwd: + pfb.fwdbwd: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-forward: + pfb.forward: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-spotless: + pfb.spotless: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image - pfb-model2comps: + pfb.model2comps: image: - _use: vars.cult-cargo.pfb-clean.image + _use: vars.cult-cargo.pfb-imaging.image From 57db05e77933d4c8fff04b8ae89862c01eb6c537 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Thu, 23 May 2024 15:45:16 +0200 Subject: [PATCH 04/21] update pfb config files --- cultcargo/genesis/pfb-imaging/latest/fastim.yaml | 3 +++ cultcargo/genesis/pfb-imaging/latest/spotless.yaml | 4 ++-- cultcargo/pfb-imaging.yml | 8 ++++---- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cultcargo/genesis/pfb-imaging/latest/fastim.yaml b/cultcargo/genesis/pfb-imaging/latest/fastim.yaml index deb20b9..f9b58bd 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fastim.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fastim.yaml @@ -146,6 +146,9 @@ inputs: dtype: bool default: true info: Display progress. Use --no-progressbar to deactivate. + output-format: + default: zarr + info: zarr or fits output _include: - (.)gridding.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml index 4c7e59e..dfdcbc1 100644 --- a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml @@ -84,8 +84,8 @@ inputs: diverge-count: dtype: int default: 5 - info: Will terminate the algorithm if the rms increases this many times. Set to - > niter to disable this check. + info: Will terminate the algorithm if the rms increases this many times. + Set to larger than niter to disable this check. field-of-view: dtype: float abbreviation: fov diff --git a/cultcargo/pfb-imaging.yml b/cultcargo/pfb-imaging.yml index 8dea45d..d2bb096 100644 --- a/cultcargo/pfb-imaging.yml +++ b/cultcargo/pfb-imaging.yml @@ -16,6 +16,10 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image + pfb.imit: + image: + _use: vars.cult-cargo.pfb-imaging.image + pfb.grid: image: _use: vars.cult-cargo.pfb-imaging.image @@ -36,10 +40,6 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image - pfb.forward: - image: - _use: vars.cult-cargo.pfb-imaging.image - pfb.spotless: image: _use: vars.cult-cargo.pfb-imaging.image From a1add018028010991e03550fff689835d3321b6f Mon Sep 17 00:00:00 2001 From: landmanbester Date: Fri, 24 May 2024 13:34:58 +0200 Subject: [PATCH 05/21] update pfb configs --- cultcargo/genesis/pfb-imaging/latest/spotless.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml index dfdcbc1..bc79b3a 100644 --- a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml @@ -84,8 +84,8 @@ inputs: diverge-count: dtype: int default: 5 - info: Will terminate the algorithm if the rms increases this many times. - Set to larger than niter to disable this check. + info: Will terminate the algorithm if the rms increases this many times. Set to + larger than niter to disable this check. field-of-view: dtype: float abbreviation: fov @@ -134,6 +134,12 @@ inputs: default: false info: Base the imaging weights on the uniform weights computed over all data instead of per band. This ensures that the MFS image can truly be uniformly weighted. + l2-reweight-from: + dtype: int + default: 10 + info: Start doing L2 reweights from this iteration onwards. L2 reweights will + also be triggered if the algorithm converges, L1 reweighting has already kicked + in and l2-reweight-dof is set. l2-reweight-dof: dtype: float default: 3.5 From e09557d83558846f9b6cb4607adb9cfe07828227 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Tue, 3 Sep 2024 11:40:01 +0200 Subject: [PATCH 06/21] merge master and add multiscale-max-scales parameters to wsclean cab --- cultcargo/genesis/wsclean/wsclean-base.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cultcargo/genesis/wsclean/wsclean-base.yml b/cultcargo/genesis/wsclean/wsclean-base.yml index 1c6240b..974cf7b 100644 --- a/cultcargo/genesis/wsclean/wsclean-base.yml +++ b/cultcargo/genesis/wsclean/wsclean-base.yml @@ -1,5 +1,5 @@ lib: - params: + params: wsclean: base-inputs: ms: @@ -70,6 +70,9 @@ lib: dtype: List[int] policies: repeat: ',' + multiscale-max-scales: + dtype: int + default: 8 multiscale-scale-bias: dtype: float scale: From 7fa03955e6eeda81a16dfd0c441e4aa53088aa31 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Tue, 3 Sep 2024 11:54:29 +0200 Subject: [PATCH 07/21] add multiscale-gain parameter to wsclean cab --- cultcargo/genesis/wsclean/wsclean-base.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cultcargo/genesis/wsclean/wsclean-base.yml b/cultcargo/genesis/wsclean/wsclean-base.yml index 974cf7b..78509e2 100644 --- a/cultcargo/genesis/wsclean/wsclean-base.yml +++ b/cultcargo/genesis/wsclean/wsclean-base.yml @@ -73,6 +73,9 @@ lib: multiscale-max-scales: dtype: int default: 8 + multiscale-gain: + dtype: float + default: 0.15 multiscale-scale-bias: dtype: float scale: From 91214e77327e6494cd88890f195212177b9b9946 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Tue, 3 Sep 2024 15:45:30 +0200 Subject: [PATCH 08/21] change channel-range repeat policy --- cultcargo/genesis/wsclean/wsclean-base.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cultcargo/genesis/wsclean/wsclean-base.yml b/cultcargo/genesis/wsclean/wsclean-base.yml index 78509e2..c4479d8 100644 --- a/cultcargo/genesis/wsclean/wsclean-base.yml +++ b/cultcargo/genesis/wsclean/wsclean-base.yml @@ -28,7 +28,7 @@ lib: channel-range: dtype: List[int] policies: - repeat: ' ' + repeat: list pol: dtype: Union[str, List[str]] element_choices: [I,Q,U,V,QU,QUV,IQ,IV,IQU,IQUV,XX,XY,YX,YY,LL,LR,RL,RR] From 7f45f8676b99e41a555546c67f9ef23c0b16ecdb Mon Sep 17 00:00:00 2001 From: landmanbester Date: Thu, 5 Sep 2024 09:52:13 +0200 Subject: [PATCH 09/21] sync latest pfb configs --- .../genesis/pfb-imaging/latest/fluxmop.yaml | 15 +- .../genesis/pfb-imaging/latest/grid.yaml | 24 +-- cultcargo/genesis/pfb-imaging/latest/hci.yaml | 157 ++++++++++++++++++ .../genesis/pfb-imaging/latest/init.yaml | 58 +++++-- .../pfb-imaging/latest/model2comps.yaml | 8 +- cultcargo/genesis/pfb-imaging/latest/out.yml | 5 + .../genesis/pfb-imaging/latest/restore.yaml | 2 +- .../genesis/pfb-imaging/latest/sara.yaml | 90 ++++++++++ .../pfb-imaging/latest/uncabbedcabs.yml | 18 +- cultcargo/genesis/pfb-imaging/sync_config.py | 2 +- 10 files changed, 336 insertions(+), 43 deletions(-) create mode 100644 cultcargo/genesis/pfb-imaging/latest/hci.yaml create mode 100644 cultcargo/genesis/pfb-imaging/latest/sara.yaml diff --git a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml index dc1a0bc..c117c6c 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml @@ -14,10 +14,15 @@ inputs: info: Either path to mask.fits or set to model to construct from model zero-model-outside-mask: dtype: bool - default: true + default: false info: Make sure the input model is zero outside the mask. Only has an effect if an external mask has been passed in. A major cycle will be triggered to recompute the residual after zeroing. + or-mask-with-model: + dtype: bool + default: false + info: Make a new mask consisting of the union of input mask and where the MFS + model is larger than zero. min-model: dtype: float default: 1e-5 @@ -31,6 +36,10 @@ inputs: dtype: str default: MODEL info: Name of the model to update + residual-name: + dtype: str + default: RESIDUAL + info: Name of the residual to use gamma: dtype: float default: 0.99 @@ -47,6 +56,10 @@ inputs: dtype: bool default: false info: Output fits cubes + memory-greedy: + dtype: bool + default: false + info: Holds data in memory if set _include: - (.)gridding.yml - (.)cgopts.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/grid.yaml b/cultcargo/genesis/pfb-imaging/latest/grid.yaml index ed2dea6..6ebe470 100644 --- a/cultcargo/genesis/pfb-imaging/latest/grid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/grid.yaml @@ -48,6 +48,10 @@ inputs: dtype: bool default: true info: Compute the PSF + noise: + dtype: bool + default: true + info: Compute noise map by sampling from weights psf-oversize: dtype: float default: 2.0 @@ -76,18 +80,11 @@ inputs: ny: dtype: int info: Number of y pixels - filter-extreme-counts: - dtype: bool - default: true - info: Replace extreme outliers in the weighting grid by their local mean - filter-nbox: - dtype: int - default: 16 - info: The size of the box to compute local mean over - filter-level: + filter-counts-level: dtype: float - default: 5 - info: The level above local mean that renders a grid weight extreme + default: 5.0 + info: Set minimum counts in the uniform weighting grid to the median divided by + this value. This is useful to avoid artificially up-weighting nearly empty uv-cells. target: dtype: str info: This can be predefined celestial objects known to astropy or a string in @@ -98,6 +95,11 @@ inputs: means no reweighting. A sensible value for this parameter depends on the level of RFI in the data. Small values (eg. 2) result in aggressive reweighting and should be avoided if the model is still incomplete. + mf-weighting: + dtype: bool + default: false + info: Base the imaging weights on the uniform weights computed over all data instead + of per band. This ensures that the MFS image can truly be uniformly weighted. _include: - (.)gridding.yml - (.)dist.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/hci.yaml b/cultcargo/genesis/pfb-imaging/latest/hci.yaml new file mode 100644 index 0000000..f9b58bd --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/hci.yaml @@ -0,0 +1,157 @@ +inputs: + ms: + dtype: URI + required: true + abbreviation: ms + info: Path to measurement set + scans: + dtype: List[int] + info: List of SCAN_NUMBERS to image. Defaults to all. Input as string eg. '[0,2]' + if running from CLI + ddids: + dtype: List[int] + info: List of DATA_DESC_ID's to images. Defaults to all. Input as string eg. '[0,1]' + if running from CLI + fields: + dtype: List[int] + info: List of FIELD_ID's to image. Defaults to all. Input as string eg. '[0,1,2]' + if running from CLI + freq-range: + dtype: str + info: Frequency range to image in Hz. Specify as a string with colon delimiter + eg. '1e9:1.1e9' + overwrite: + dtype: bool + default: false + info: Allow overwrite of output xds + transfer-model-from: + dtype: str + abbreviation: tmf + info: Name of dataset to use for model initialisation + data-column: + dtype: str + default: DATA + abbreviation: dc + info: Data column to image. Must be the same across MSs + weight-column: + dtype: str + abbreviation: wc + info: Column containing natural weights. Must be the same across MSs + sigma-column: + dtype: str + abbreviation: sc + info: Column containing standard devations. Will be used to initialise natural + weights if detected. Must be the same across MSs + flag-column: + dtype: str + default: FLAG + abbreviation: fc + info: Column containing data flags. Must be the same across MSs + gain-table: + dtype: URI + info: Path to Quartical gain table containing NET gains. + integrations-per-image: + dtype: int + abbreviation: ipi + default: -1 + info: Number of time integrations per image. Default (-1, 0, None) -> dataset + per scan. + channels-per-degrid-image: + dtype: int + abbreviation: cpdi + default: 32 + info: Number of channels per image for degridding resolution. Any of (-1, 0, None) + implies single dataset per spw. + channels-per-grid-image: + dtype: int + abbreviation: cpgi + default: -1 + info: Number of channels per image for gridding resolution. Default of (-1, 0, + None) implies a single dataset per spw. + precision: + dtype: str + default: double + choices: + - single + - double + info: Gridding precision + field-of-view: + dtype: float + abbreviation: fov + default: 3.0 + info: Field of view in degrees + super-resolution-factor: + dtype: float + default: 2 + abbreviation: srf + info: Will over-sample Nyquist by this factor at max frequency + cell-size: + dtype: float + info: Cell size in arc-seconds + nx: + dtype: int + info: Number of x pixels + ny: + dtype: int + info: Number of y pixels + robustness: + dtype: float + abbreviation: rob + info: Robustness factor for Briggs weighting. None means natural + filter-extreme-counts: + dtype: bool + default: false + info: Replace extreme outliers in the weighting grid by their local mean + filter-nbox: + dtype: int + default: 16 + info: The size of the box to compute local mean over + filter-level: + dtype: float + default: 10 + info: The level above local mean that renders a grid weight extreme + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + l2reweight-dof: + dtype: float + info: The degrees of freedom parameter for L2 reweighting. The default (None) + means no reweighting. A sensible value for this parameter depends on the level + of RFI in the data. Small values (eg. 2) result in aggressive reweighting and + should be avoided if the model is still incomplete. + host-address: + dtype: str + abbreviation: ha + info: Address where the distributed client lives. Uses LocalCluster if no address + is provided and scheduler is set to distributed. + nworkers: + dtype: int + default: 1 + abbreviation: nw + info: Number of worker processes. Use with distributed scheduler. + nthreads-dask: + dtype: int + default: 1 + abbreviation: ntd + info: Number of dask threads. This will largely determine the memory footprint. + Used as nthreads per worker when using distributed scheduler. + nvthreads: + dtype: int + abbreviation: nvt + info: Number of threads used to scale vertically (eg. for FFTs and gridding). + Each dask thread can in principle spawn this many threads. Will attempt to use + half the available threads by default. + progressbar: + dtype: bool + default: true + info: Display progress. Use --no-progressbar to deactivate. + output-format: + default: zarr + info: zarr or fits output + + _include: + - (.)gridding.yml + - (.)out.yml + +outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/init.yaml b/cultcargo/genesis/pfb-imaging/latest/init.yaml index 5639c1f..92955c8 100644 --- a/cultcargo/genesis/pfb-imaging/latest/init.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/init.yaml @@ -6,16 +6,16 @@ inputs: info: Path to measurement set scans: dtype: List[int] - info: List of SCAN_NUMBERS to image. Defaults to all. Input as string eg. '[0,2]' - if running from CLI + info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated + list 0,2 if running from CLI ddids: dtype: List[int] - info: List of DATA_DESC_ID's to images. Defaults to all. Input as string eg. '[0,1]' - if running from CLI + info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated + list 0,2 if running from CLI fields: dtype: List[int] - info: List of FIELD_ID's to image. Defaults to all. Input as string eg. '[0,1,2]' - if running from CLI + info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list + 0,2 if running from CLI freq-range: dtype: str info: Frequency range to image in Hz. Specify as a string with colon delimiter @@ -49,7 +49,7 @@ inputs: gain-table: dtype: List[str] info: Path to Quartical gain table containing NET gains. There must be a table - for each MS and glob(ms) and glob(gt) should match up + for each MS and glob(ms) and glob(gt) should match up when running from CLI. integrations-per-image: dtype: int abbreviation: ipi @@ -59,8 +59,9 @@ inputs: channels-per-image: dtype: int abbreviation: cpi - default: -1 - info: Number of channels per image. Default (-1, 0, None) -> dataset per spw. + default: 32 + info: Number of channels per image for degridding resolution. Any of (-1, 0, None) + implies single dataset per spw. precision: dtype: str default: double @@ -75,22 +76,43 @@ inputs: beam-model: dtype: str info: Beam model to use - max-field-of-view: - dtype: float - abbreviation: fov - default: 3.0 - info: Field of view in degrees chan-average: dtype: int default: 1 info: Average this number if channels together - visualize-graph: + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + progressbar: dtype: bool - default: false - info: Visualise the compute graph. + default: true + info: Display progress. Use --no-progressbar to deactivate. + host-address: + dtype: str + abbreviation: ha + info: Address where the distributed client lives. Uses LocalCluster if no address + is provided and scheduler is set to distributed. + nworkers: + dtype: int + default: 1 + abbreviation: nw + info: Number of worker processes. This will largely determine the memory footprint. + nthreads-dask: + dtype: int + default: 1 + abbreviation: ntd + info: Number of dask threads per worker. Only change this if you know what you + are doing. + nvthreads: + dtype: int + default: 1 + abbreviation: nvt + info: Number of threads used to scale vertically (eg. for FFTs and gridding). + Each dask thread can in principle spawn this many threads. Will attempt to use + half the available threads by default. _include: - - (.)dist.yml - (.)out.yml outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml index 80b8ec7..7e552bc 100644 --- a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml @@ -5,8 +5,8 @@ inputs: info: Allow overwrite of existing model nbasisf: dtype: int - default: 4 - info: Order of interpolating polynomial for frequency axis + info: Order of interpolating polynomial for frequency axis. One less than the + number of bands by default. nbasist: dtype: int default: 1 @@ -49,6 +49,10 @@ inputs: - zarr - json info: Format to dump model to. + out-freqs: + dtype: str + info: A string flow:fhigh:step of frequencies in hertz where the output cube needs + to be evaluated. _include: - (.)dist.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/out.yml b/cultcargo/genesis/pfb-imaging/latest/out.yml index 09c02c4..fb87190 100644 --- a/cultcargo/genesis/pfb-imaging/latest/out.yml +++ b/cultcargo/genesis/pfb-imaging/latest/out.yml @@ -14,3 +14,8 @@ product: default: I info: Imaging products to produce. Options are I, Q, U, V. Only single Stokes products are currently supported +fits-output-folder: + dtype: str + abbreviation: fitso + info: Optional path to write fits files to. Set to output-filename if not provided. + The same naming conventions apply. diff --git a/cultcargo/genesis/pfb-imaging/latest/restore.yaml b/cultcargo/genesis/pfb-imaging/latest/restore.yaml index 1e6bad2..1ce4480 100644 --- a/cultcargo/genesis/pfb-imaging/latest/restore.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/restore.yaml @@ -29,7 +29,7 @@ inputs: cubes. overwrite: dtype: bool - default: false + default: true info: Allow overwriting fits files outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/sara.yaml b/cultcargo/genesis/pfb-imaging/latest/sara.yaml new file mode 100644 index 0000000..20cbab0 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/sara.yaml @@ -0,0 +1,90 @@ +inputs: + _include: + - (.)gridding.yml + - (.)pdopts.yml + - (.)pmopts.yml + - (.)dist.yml + - (.)out.yml + nband: + dtype: int + required: true + abbreviation: nb + info: Number of imaging bands + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products + bases: + dtype: str + info: 'Wavelet bases to use. Give as comma separated str eg. ' + default: self,db1,db2 + abbreviation: bases + nlevels: + dtype: int + info: Wavelet decomposition level + default: 3 + abbreviation: nlevels + l1reweight-from: + dtype: int + default: 5 + info: L1 reweighting will kick in either at convergence or after this many iterations. + Set to a negative value to disbale L1 reweighting. + hessnorm: + dtype: float + abbreviation: hessnorm + info: Spectral norm of hessian approximation + rmsfactor: + dtype: float + default: 1.0 + info: By default will threshold by rmsfactor*rms at every iteration + gamma: + dtype: float + default: 1.0 + info: Step size of update + alpha: + dtype: float + default: 2 + info: Controls how aggressively the l1reweighting is applied. Larger values correspond + to more agressive reweighting. + positivity: + dtype: int + default: 1 + # choices: + # - 0 + # - 1 + # - 2 + info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity + constraint and 2 -> strong positivity i.e. all pixels in a band > 0 + niter: + dtype: int + default: 10 + abbreviation: niter + info: Number of iterations. + tol: + dtype: float + default: 5e-4 + info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) + < tol + fits-mfs: + dtype: bool + default: true + info: Output MFS fits files + fits-cubes: + dtype: bool + default: false + info: Output fits cubes + memory-greedy: + dtype: bool + default: false + info: Holds data in memory if set + init-factor: + dtype: float + default: 0.5 + info: Lower the initial rmsfactor by this amount + diverge-count: + dtype: int + default: 5 + info: Will terminate the algorithm if the rms increases this many times. Set to + larger than niter to disable this check. + +outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml index 55bc110..b62b272 100644 --- a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml +++ b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml @@ -7,15 +7,6 @@ pfb.init: _include: - (.)init.yaml -pfb.imit: - command: pfb.workers.imit.imit - flavour: python - policies: - pass_missing_as_none: true - - _include: - - (.)imit.yaml - pfb.grid: command: pfb.workers.grid.grid flavour: python @@ -79,6 +70,15 @@ pfb.spotless: _include: - (.)spotless.yaml +pfb.sara: + command: pfb.workers.sara.sara + flavour: python + policies: + pass_missing_as_none: true + + _include: + - (.)sara.yaml + pfb.model2comps: command: pfb.workers.model2comps.model2comps flavour: python diff --git a/cultcargo/genesis/pfb-imaging/sync_config.py b/cultcargo/genesis/pfb-imaging/sync_config.py index 25adc2f..5633b80 100644 --- a/cultcargo/genesis/pfb-imaging/sync_config.py +++ b/cultcargo/genesis/pfb-imaging/sync_config.py @@ -11,7 +11,7 @@ content_path = 'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/band_actors/pfb/parser' # returns list of files in subfolder -response = requests.get('https://api.github.com/repos/ratt-ru/pfb-imaging/contents/pfb/parser?ref=band_actors') +response = requests.get('https://api.github.com/repos/ratt-ru/pfb-imaging/contents/pfb/parser?ref=outputs') for r in response.json(): name = r['name'] print(name) From 77c8883e80e11f06a59ed6b2289f373f96132513 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Thu, 5 Sep 2024 09:58:12 +0200 Subject: [PATCH 10/21] sync pfb configs from outputs branch --- .../genesis/pfb-imaging/latest/cgopts.yml | 2 +- .../genesis/pfb-imaging/latest/degrid.yaml | 8 ++ cultcargo/genesis/pfb-imaging/latest/dist.yml | 23 ++-- .../genesis/pfb-imaging/latest/fluxmop.yaml | 24 ++--- .../genesis/pfb-imaging/latest/fwdbwd.yaml | 13 +-- .../genesis/pfb-imaging/latest/grid.yaml | 30 ++---- cultcargo/genesis/pfb-imaging/latest/hci.yaml | 102 ++++++++---------- .../genesis/pfb-imaging/latest/init.yaml | 53 ++++----- .../genesis/pfb-imaging/latest/klean.yaml | 26 ++--- .../pfb-imaging/latest/model2comps.yaml | 15 ++- cultcargo/genesis/pfb-imaging/latest/out.yml | 9 +- .../genesis/pfb-imaging/latest/pdopts.yml | 4 +- .../genesis/pfb-imaging/latest/restore.yaml | 25 ++++- .../genesis/pfb-imaging/latest/sara.yaml | 65 ++++++----- .../genesis/pfb-imaging/latest/smoovie.yaml | 8 +- .../genesis/pfb-imaging/latest/spotless.yaml | 89 +++++++-------- .../pfb-imaging/latest/uncabbedcabs.yml | 18 ++-- cultcargo/genesis/pfb-imaging/sync_config.py | 6 +- 18 files changed, 253 insertions(+), 267 deletions(-) diff --git a/cultcargo/genesis/pfb-imaging/latest/cgopts.yml b/cultcargo/genesis/pfb-imaging/latest/cgopts.yml index eafdc63..3f3cc55 100644 --- a/cultcargo/genesis/pfb-imaging/latest/cgopts.yml +++ b/cultcargo/genesis/pfb-imaging/latest/cgopts.yml @@ -26,5 +26,5 @@ cg-report-freq: abbreviation: cgrf backtrack: dtype: bool - default: true + default: false info: Ensure residual decreases at every iteration diff --git a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml index 50c2dd6..45e8adc 100644 --- a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml @@ -19,6 +19,10 @@ inputs: default: I info: Imaging products to degrid. Options are I, Q, U, V. Only single Stokes products are currently supported. + freq-range: + dtype: str + info: Frequency range to image in Hz. Specify as a string with colon delimiter + eg. '1e9:1.1e9' integrations-per-image: dtype: int default: -1 @@ -38,5 +42,9 @@ inputs: _include: - (.)gridding.yml - (.)dist.yml + - (.)out.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/dist.yml b/cultcargo/genesis/pfb-imaging/latest/dist.yml index 2739012..397b022 100644 --- a/cultcargo/genesis/pfb-imaging/latest/dist.yml +++ b/cultcargo/genesis/pfb-imaging/latest/dist.yml @@ -8,21 +8,18 @@ nworkers: default: 1 abbreviation: nw info: Number of worker processes. Use with distributed scheduler. -nthreads-dask: +nthreads: dtype: int - default: 1 - abbreviation: ntd - info: Number of dask threads. This will largely determine the memory footprint. - Used as nthreads per worker when using distributed scheduler. -nvthreads: - dtype: int - abbreviation: nvt + abbreviation: nt info: Number of threads used to scale vertically (eg. for FFTs and gridding). Each dask thread can in principle spawn this many threads. Will attempt to use half the available threads by default. -scheduler: +direct_to_workers: + dtype: bool + default: true + info: Connect direct to workers i.e. bypass scheduler. Faster but then the dashboard + isn't very useful. +log-level: dtype: str - default: threads - choices: [threads, single-threaded, sync, distributed, processes] - abbreviation: scheduler - info: Which dask scheduler to use. + default: error + choices: [error, warning, info, debug] diff --git a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml index c117c6c..8cfe0c5 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml @@ -1,9 +1,4 @@ inputs: - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands suffix: dtype: str default: main @@ -27,11 +22,15 @@ inputs: dtype: float default: 1e-5 info: If using mask to construct model construct it where model > min-model - sigmainv: + sigmainvsq: dtype: float default: 1e-5 abbreviation: sinv info: Standard deviation of assumed GRF prior + sigma: + dtype: float + default: 1 + info: The value that is added to |psfhat| for preconditioning. model-name: dtype: str default: MODEL @@ -42,20 +41,12 @@ inputs: info: Name of the residual to use gamma: dtype: float - default: 0.99 + default: 1.0 info: Step size of update use-psf: dtype: bool default: true info: Whether to approximate the Hessian as a convolution by the PSF - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes memory-greedy: dtype: bool default: false @@ -67,3 +58,6 @@ inputs: - (.)out.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml b/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml index 18a5a72..fe18f62 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml @@ -44,7 +44,7 @@ inputs: dtype: float abbreviation: sig21 info: Wavelet thresholding level - l1reweight-from: + l1-reweight-from: dtype: int default: 5 info: Start doing l1reweights after this many iterations @@ -75,14 +75,6 @@ inputs: default: 1e-5 info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) < tol - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes memory-greedy: dtype: bool default: false @@ -98,3 +90,6 @@ inputs: to the dirty image outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/grid.yaml b/cultcargo/genesis/pfb-imaging/latest/grid.yaml index 6ebe470..c217f0a 100644 --- a/cultcargo/genesis/pfb-imaging/latest/grid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/grid.yaml @@ -1,11 +1,6 @@ inputs: xds: info: Optional explicit path to xds. Set using output-filename and suffix by default. - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands suffix: dtype: str default: main @@ -32,14 +27,6 @@ inputs: dtype: float abbreviation: rob info: Robustness factor for Briggs weighting. None means natural - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes dirty: dtype: bool default: true @@ -48,13 +35,17 @@ inputs: dtype: bool default: true info: Compute the PSF + residual: + dtype: bool + default: true + info: Compute the residual (only if model is present) noise: dtype: bool default: true info: Compute noise map by sampling from weights psf-oversize: dtype: float - default: 2.0 + default: 1.0 abbreviation: psfo info: Size of PSF relative to dirty image weight: @@ -89,20 +80,19 @@ inputs: dtype: str info: This can be predefined celestial objects known to astropy or a string in the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - l2reweight-dof: + l2-reweight-dof: dtype: float + abbreviation: dof info: The degrees of freedom parameter for L2 reweighting. The default (None) means no reweighting. A sensible value for this parameter depends on the level of RFI in the data. Small values (eg. 2) result in aggressive reweighting and should be avoided if the model is still incomplete. - mf-weighting: - dtype: bool - default: false - info: Base the imaging weights on the uniform weights computed over all data instead - of per band. This ensures that the MFS image can truly be uniformly weighted. _include: - (.)gridding.yml - (.)dist.yml - (.)out.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/hci.yaml b/cultcargo/genesis/pfb-imaging/latest/hci.yaml index f9b58bd..b679310 100644 --- a/cultcargo/genesis/pfb-imaging/latest/hci.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/hci.yaml @@ -1,21 +1,21 @@ inputs: ms: - dtype: URI + dtype: List[URI] required: true abbreviation: ms info: Path to measurement set scans: dtype: List[int] - info: List of SCAN_NUMBERS to image. Defaults to all. Input as string eg. '[0,2]' - if running from CLI + info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated + list 0,2 if running from CLI ddids: dtype: List[int] - info: List of DATA_DESC_ID's to images. Defaults to all. Input as string eg. '[0,1]' - if running from CLI + info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated + list 0,2 if running from CLI fields: dtype: List[int] - info: List of FIELD_ID's to image. Defaults to all. Input as string eg. '[0,1,2]' - if running from CLI + info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list + 0,2 if running from CLI freq-range: dtype: str info: Frequency range to image in Hz. Specify as a string with colon delimiter @@ -32,7 +32,14 @@ inputs: dtype: str default: DATA abbreviation: dc - info: Data column to image. Must be the same across MSs + info: Data column to image. Must be the same across MSs. Simple arithmetic is + supported eg. 'CORRECTED_DATA-MODEL_DATA'. When gains are present this column + will be corrected. + model-column: + dtype: str + abbreviation: mc + info: Model column to subtract from corrected data column. Useful to avoid high + time res degridding. weight-column: dtype: str abbreviation: wc @@ -48,23 +55,18 @@ inputs: abbreviation: fc info: Column containing data flags. Must be the same across MSs gain-table: - dtype: URI - info: Path to Quartical gain table containing NET gains. + dtype: List[URI] + info: Path to Quartical gain table containing NET gains. There must be a table + for each MS and glob(ms) and glob(gt) should match up when running from CLI. integrations-per-image: dtype: int abbreviation: ipi - default: -1 + default: 1 info: Number of time integrations per image. Default (-1, 0, None) -> dataset per scan. - channels-per-degrid-image: + channels-per-image: dtype: int - abbreviation: cpdi - default: 32 - info: Number of channels per image for degridding resolution. Any of (-1, 0, None) - implies single dataset per spw. - channels-per-grid-image: - dtype: int - abbreviation: cpgi + abbreviation: cpi default: -1 info: Number of channels per image for gridding resolution. Default of (-1, 0, None) implies a single dataset per spw. @@ -78,11 +80,11 @@ inputs: field-of-view: dtype: float abbreviation: fov - default: 3.0 + default: 1.0 info: Field of view in degrees super-resolution-factor: dtype: float - default: 2 + default: 1.4 abbreviation: srf info: Will over-sample Nyquist by this factor at max frequency cell-size: @@ -98,50 +100,16 @@ inputs: dtype: float abbreviation: rob info: Robustness factor for Briggs weighting. None means natural - filter-extreme-counts: - dtype: bool - default: false - info: Replace extreme outliers in the weighting grid by their local mean - filter-nbox: - dtype: int - default: 16 - info: The size of the box to compute local mean over - filter-level: - dtype: float - default: 10 - info: The level above local mean that renders a grid weight extreme target: dtype: str info: This can be predefined celestial objects known to astropy or a string in the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - l2reweight-dof: + l2-reweight-dof: dtype: float info: The degrees of freedom parameter for L2 reweighting. The default (None) means no reweighting. A sensible value for this parameter depends on the level of RFI in the data. Small values (eg. 2) result in aggressive reweighting and should be avoided if the model is still incomplete. - host-address: - dtype: str - abbreviation: ha - info: Address where the distributed client lives. Uses LocalCluster if no address - is provided and scheduler is set to distributed. - nworkers: - dtype: int - default: 1 - abbreviation: nw - info: Number of worker processes. Use with distributed scheduler. - nthreads-dask: - dtype: int - default: 1 - abbreviation: ntd - info: Number of dask threads. This will largely determine the memory footprint. - Used as nthreads per worker when using distributed scheduler. - nvthreads: - dtype: int - abbreviation: nvt - info: Number of threads used to scale vertically (eg. for FFTs and gridding). - Each dask thread can in principle spawn this many threads. Will attempt to use - half the available threads by default. progressbar: dtype: bool default: true @@ -149,9 +117,31 @@ inputs: output-format: default: zarr info: zarr or fits output + sigmainvsq: + dtype: float + default: 1e-5 + info: Value to add to Hessian to make it invertible. Smaller values tend to fit + more noise and make the inversion less stable. + concat-time: + dtype: bool + default: false + info: Concatenate gains and times across scan + natural-grad: + dtype: bool + default: false + info: Compute naural gradient + memory_reporting: + dtype: bool + default: false + info: Report worker memory as tasks complete _include: - (.)gridding.yml - (.)out.yml + - (.)dist.yml + - (.)cgopts.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/init.yaml b/cultcargo/genesis/pfb-imaging/latest/init.yaml index 92955c8..b9f5759 100644 --- a/cultcargo/genesis/pfb-imaging/latest/init.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/init.yaml @@ -5,19 +5,19 @@ inputs: abbreviation: ms info: Path to measurement set scans: - dtype: List[int] + dtype: Optional[List[int]] info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated list 0,2 if running from CLI ddids: - dtype: List[int] + dtype: Optional[List[int]] info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated list 0,2 if running from CLI fields: - dtype: List[int] + dtype: Optional[List[int]] info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list 0,2 if running from CLI freq-range: - dtype: str + dtype: Optional[str] info: Frequency range to image in Hz. Specify as a string with colon delimiter eg. '1e9:1.1e9' overwrite: @@ -31,7 +31,9 @@ inputs: dtype: str default: DATA abbreviation: dc - info: Data column to image. Must be the same across MSs + info: Data column to image. Must be the same across MSs. Simple arithmetic is + supported eg. 'CORRECTED_DATA-MODEL_DATA'. When gains are present this column + will be corrected. weight-column: dtype: str abbreviation: wc @@ -47,7 +49,7 @@ inputs: abbreviation: fc info: Column containing data flags. Must be the same across MSs gain-table: - dtype: List[str] + dtype: List[URI] info: Path to Quartical gain table containing NET gains. There must be a table for each MS and glob(ms) and glob(gt) should match up when running from CLI. integrations-per-image: @@ -73,9 +75,15 @@ inputs: dtype: float default: 1.0 info: BDA decorrelation factor. Only has an effect if less than one + max-field-of-view: + dtype: float + default: 3.0 + abbreviation: max-fov + info: The maximum field of view that will be considered. Used to compute decorrelation + due to BDA. beam-model: dtype: str - info: Beam model to use + info: Beam model to use. Only katbeam currently supported. chan-average: dtype: int default: 1 @@ -88,31 +96,16 @@ inputs: dtype: bool default: true info: Display progress. Use --no-progressbar to deactivate. - host-address: - dtype: str - abbreviation: ha - info: Address where the distributed client lives. Uses LocalCluster if no address - is provided and scheduler is set to distributed. - nworkers: - dtype: int - default: 1 - abbreviation: nw - info: Number of worker processes. This will largely determine the memory footprint. - nthreads-dask: - dtype: int - default: 1 - abbreviation: ntd - info: Number of dask threads per worker. Only change this if you know what you - are doing. - nvthreads: - dtype: int - default: 1 - abbreviation: nvt - info: Number of threads used to scale vertically (eg. for FFTs and gridding). - Each dask thread can in principle spawn this many threads. Will attempt to use - half the available threads by default. + memory_reporting: + dtype: bool + default: false + info: Report worker memory as tasks complete _include: - (.)out.yml + - (.)dist.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/klean.yaml b/cultcargo/genesis/pfb-imaging/latest/klean.yaml index 3213b32..fe0869f 100644 --- a/cultcargo/genesis/pfb-imaging/latest/klean.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/klean.yaml @@ -1,9 +1,4 @@ inputs: - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands suffix: dtype: str default: main @@ -16,14 +11,6 @@ inputs: dtype: int default: 1 info: Perform dilation followed by erosion with structure element of this width - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes mop-flux: dtype: bool default: true @@ -38,6 +25,10 @@ inputs: default: 5 abbreviation: niter info: Number of major iterations + nthreads: + dtype: int + abbreviation: nt + info: Total number of threads to use. Defaults to half the total number available. threshold: dtype: float info: Absolute threshold at which to stop cleaning. By default it is set automatically @@ -112,13 +103,12 @@ inputs: dtype: bool default: true info: Ensure residual decreases at every iteration - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set + _include: - (.)gridding.yml - - (.)dist.yml - (.)out.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml index 7e552bc..fe8229a 100644 --- a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml @@ -3,6 +3,10 @@ inputs: dtype: bool default: false info: Allow overwrite of existing model + mds: + dtype: str + info: An optional input model to append to the model. Will be rendered to the + same resolution as the model in the dds. nbasisf: dtype: int info: Order of interpolating polynomial for frequency axis. One less than the @@ -12,6 +16,10 @@ inputs: default: 1 info: Order of interpolating polynomial for time axis. This is hypothetical for the time being. + nthreads: + dtype: int + abbreviation: nt + info: Number of threads fit-mode: dtype: str default: Legendre @@ -27,9 +35,6 @@ inputs: default: MODEL abbreviation: mname info: Name of model in mds - model-fits: - dtype: str - info: Specify to degrid from a fits file use-wsum: dtype: bool default: true @@ -55,7 +60,9 @@ inputs: to be evaluated. _include: - - (.)dist.yml - (.)out.yml outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/out.yml b/cultcargo/genesis/pfb-imaging/latest/out.yml index fb87190..aeb7b1c 100644 --- a/cultcargo/genesis/pfb-imaging/latest/out.yml +++ b/cultcargo/genesis/pfb-imaging/latest/out.yml @@ -5,7 +5,6 @@ output-filename: info: Basename of output log-directory: dtype: str - required: true abbreviation: ldir info: Directory to write logs and performance reports to. product: @@ -19,3 +18,11 @@ fits-output-folder: abbreviation: fitso info: Optional path to write fits files to. Set to output-filename if not provided. The same naming conventions apply. +fits-mfs: + dtype: bool + default: true + info: Output MFS fits files +fits-cubes: + dtype: bool + default: true + info: Output fits cubes diff --git a/cultcargo/genesis/pfb-imaging/latest/pdopts.yml b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml index 7e0db69..1006587 100644 --- a/cultcargo/genesis/pfb-imaging/latest/pdopts.yml +++ b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml @@ -1,6 +1,6 @@ pd-tol: - dtype: float - default: 3e-4 + dtype: List[float] + default: [3e-4] info: Tolreance of primal dual algorithm abbreviation: pdtol pd-maxit: diff --git a/cultcargo/genesis/pfb-imaging/latest/restore.yaml b/cultcargo/genesis/pfb-imaging/latest/restore.yaml index 1ce4480..0a9f5ab 100644 --- a/cultcargo/genesis/pfb-imaging/latest/restore.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/restore.yaml @@ -12,11 +12,6 @@ inputs: abbreviation: rname default: RESIDUAL info: Name of residual in dds - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands suffix: dtype: str default: main @@ -31,5 +26,25 @@ inputs: dtype: bool default: true info: Allow overwriting fits files + gausspar: + dtype: List[float] + info: Gaussian parameters (e-major, e-minor, position-angle) specifying the resolution + to restore images to. The major and minor axes need to be specified in units + of arcseconds and the position-angle in degrees. The default resolution is the + native resolution in each imaging band. This parameter can be used to homogenise + the resolution of the cubes. Set to (0,0,0) to use the resolution of the lowest + band. + inflate-factor: + dtype: float + default: 1.5 + info: Inflate the intrinsic resolution of the uniformly blurred image by this + amount. + drop-bands: + dtype: List[int] + # default: [] How to make the default an empty list? + info: List of bands to discard outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/sara.yaml b/cultcargo/genesis/pfb-imaging/latest/sara.yaml index 20cbab0..5e948c0 100644 --- a/cultcargo/genesis/pfb-imaging/latest/sara.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/sara.yaml @@ -3,13 +3,7 @@ inputs: - (.)gridding.yml - (.)pdopts.yml - (.)pmopts.yml - - (.)dist.yml - (.)out.yml - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands suffix: dtype: str default: main @@ -17,26 +11,37 @@ inputs: bases: dtype: str info: 'Wavelet bases to use. Give as comma separated str eg. ' - default: self,db1,db2 + default: self,db1,db2,db3 abbreviation: bases nlevels: dtype: int info: Wavelet decomposition level - default: 3 + default: 2 abbreviation: nlevels - l1reweight-from: + l1-reweight-from: dtype: int default: 5 info: L1 reweighting will kick in either at convergence or after this many iterations. Set to a negative value to disbale L1 reweighting. - hessnorm: + hess-norm: dtype: float - abbreviation: hessnorm + abbreviation: hess-norm info: Spectral norm of hessian approximation + hess-approx: + dtype: str + default: direct + abbreviation: hess-approx + choices: [wgt, psf, direct] + info: Which Hessian approximation to use. Set to wgt for vis space approximation, + psf for zero-padded image space approximation and direct for direct inversion. rmsfactor: dtype: float default: 1.0 info: By default will threshold by rmsfactor*rms at every iteration + epsfactor: + dtype: float + default: 1.0 + info: Will use epsfactor*wsum to regularise the inversion of the Hessian approximation. gamma: dtype: float default: 1.0 @@ -46,6 +51,10 @@ inputs: default: 2 info: Controls how aggressively the l1reweighting is applied. Larger values correspond to more agressive reweighting. + nbasisf: + dtype: int + info: Number of basis functions to use while fitting the frequency axis. Default + is to use the number of non-null imaging bands i.e. interpolation. positivity: dtype: int default: 1 @@ -60,31 +69,35 @@ inputs: default: 10 abbreviation: niter info: Number of iterations. + nthreads: + dtype: int + abbreviation: nt + info: Total number of threads to use. Defaults to half the total number available. tol: dtype: float default: 5e-4 info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) < tol - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: + diverge-count: + dtype: int + default: 5 + info: Will terminate the algorithm if the rms increases this many times. Set to + larger than niter to disable this check. + skip-model: dtype: bool default: false - info: Output fits cubes - memory-greedy: + info: Skip model creation i.e. use with do-res to do-res only + rms-outside-model: dtype: bool default: false - info: Holds data in memory if set - init-factor: + info: Mask residual where model is non-zero when computing rms. This is not recommended + for largely non-empty fields. + init_factor: dtype: float default: 0.5 - info: Lower the initial rmsfactor by this amount - diverge-count: - dtype: int - default: 5 - info: Will terminate the algorithm if the rms increases this many times. Set to - larger than niter to disable this check. + info: Reduce the regularisation strength by this fraction at the outset. outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml index c67ecf5..4ef4178 100644 --- a/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml @@ -89,9 +89,8 @@ inputs: nthreads: dtype: int default: 1 - abbreviation: ntd - info: Number of dask threads. This will largely determine the memory footprint. - Used as nthreads per worker when using distributed scheduler. + abbreviation: nt + info: Number of threads. # TBD field-of-view: dtype: float @@ -130,3 +129,6 @@ inputs: info: Constant rate factor for controlling mp4 output quality. outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml index bc79b3a..945fe88 100644 --- a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml @@ -3,17 +3,24 @@ inputs: - (.)gridding.yml - (.)pdopts.yml - (.)pmopts.yml + - (.)cgopts.yml - (.)dist.yml - (.)out.yml - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands + xds: + info: Optional explicit path to xds. Set using output-filename and product by + default. + mds: + info: Optional explicit path to mds. Set using output-filename, product and suffix + by default. suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products + robustness: + dtype: float + abbreviation: rob + default: 0.5 + info: Robustness factor for Briggs weighting. None means natural bases: dtype: str info: 'Wavelet bases to use. Give as comma separated str eg. ' @@ -24,15 +31,22 @@ inputs: info: Wavelet decomposition level default: 3 abbreviation: nlevels - l1reweight-from: + l1-reweight-from: dtype: int default: 5 info: L1 reweighting will kick in either at convergence or after this many iterations. Set to a negative value to disbale L1 reweighting. - hessnorm: + hess-norm: dtype: float - abbreviation: hessnorm + abbreviation: hess-norm info: Spectral norm of hessian approximation + hess-approx: + dtype: str + default: direct + abbreviation: hess-approx + choices: [wgt, psf, direct] + info: Which Hessian approximation to use. Set to wgt for vis space approximation, + psf for zero-padded image space approximation and direct for direct inversion. rmsfactor: dtype: float default: 1.0 @@ -49,12 +63,12 @@ inputs: positivity: dtype: int default: 1 - # choices: - # - 0 - # - 1 - # - 2 - info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity - constraint and 2 -> strong positivity i.e. all pixels in a band > 0 + # choices: [0,1,2] + info: | + How to apply positivity constraint + 0 -> no positivity + 1 -> normal positivity constraint + 2 -> strong positivity i.e. all pixels in a band > 0 niter: dtype: int default: 10 @@ -65,22 +79,6 @@ inputs: default: 5e-4 info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) < tol - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set - init-factor: - dtype: float - default: 0.5 - info: Lower the initial rmsfactor by this amount diverge-count: dtype: int default: 5 @@ -105,35 +103,15 @@ inputs: ny: dtype: int info: Number of y pixels - robustness: - dtype: float - default: 0.0 - abbreviation: rob - info: Robustness value for Briggs weighting psf-oversize: dtype: float - default: 2.0 + default: 1.0 abbreviation: psfo info: Size of PSF relative to dirty image - filter-counts-level: - dtype: float - default: 5.0 - info: Set minimum counts in the uniform weighting grid to the median divided by - this value. This is useful to avoid artificially up-weighting nearly empty uv-cells. target: dtype: str info: This can be predefined celestial objects known to astropy or a string in the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - concat: - dtype: bool - default: true - info: Concatenate datasets by row/time. This will result in a single dirty image, - psf and beam per band. - mf-weighting: - dtype: bool - default: false - info: Base the imaging weights on the uniform weights computed over all data instead - of per band. This ensures that the MFS image can truly be uniformly weighted. l2-reweight-from: dtype: int default: 10 @@ -150,9 +128,16 @@ inputs: dtype: int default: 5 info: The maximum number of L2 reweighting steps. - reset-cache: + overwrite: dtype: bool default: false info: Hard reset the cache + sigmainvsq: + dtype: float + default: 1.0 + info: Value to add to Hessian to make it invertible outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml index b62b272..91590c2 100644 --- a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml +++ b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml @@ -43,15 +43,6 @@ pfb.restore: _include: - (.)restore.yaml -pfb.fwdbwd: - command: pfb.workers.fwdbwd.fwdbwd - flavour: python - policies: - pass_missing_as_none: true - - _include: - - (.)fwdbwd.yaml - pfb.fluxmop: command: pfb.workers.fluxmop.fluxmop flavour: python @@ -87,3 +78,12 @@ pfb.model2comps: _include: - (.)model2comps.yaml + +pfb.hci: + command: pfb.workers.hci.hci + flavour: python + policies: + pass_missing_as_none: true + + _include: + - (.)hci.yaml diff --git a/cultcargo/genesis/pfb-imaging/sync_config.py b/cultcargo/genesis/pfb-imaging/sync_config.py index 5633b80..5bfb478 100644 --- a/cultcargo/genesis/pfb-imaging/sync_config.py +++ b/cultcargo/genesis/pfb-imaging/sync_config.py @@ -7,11 +7,11 @@ mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-imaging/latest' -# import ipdb; ipdb.set_trace() -content_path = 'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/band_actors/pfb/parser' +branch = 'outputs' +content_path = f'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/{branch}/pfb/parser' # returns list of files in subfolder -response = requests.get('https://api.github.com/repos/ratt-ru/pfb-imaging/contents/pfb/parser?ref=outputs') +response = requests.get(f'https://api.github.com/repos/ratt-ru/pfb-imaging/contents/pfb/parser?ref={branch}') for r in response.json(): name = r['name'] print(name) From 5da5b6b4b18aec261832a202ec67d95cfdfaf0c2 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Thu, 5 Sep 2024 10:02:44 +0200 Subject: [PATCH 11/21] update stimela cab definitions --- .../genesis/pfb-imaging/latest/fastim.yaml | 157 ------------------ .../genesis/pfb-imaging/latest/fwdbwd.yaml | 95 ----------- .../genesis/pfb-imaging/latest/imit.yaml | 97 ----------- cultcargo/pfb-imaging.yml | 14 +- 4 files changed, 9 insertions(+), 354 deletions(-) delete mode 100644 cultcargo/genesis/pfb-imaging/latest/fastim.yaml delete mode 100644 cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml delete mode 100644 cultcargo/genesis/pfb-imaging/latest/imit.yaml diff --git a/cultcargo/genesis/pfb-imaging/latest/fastim.yaml b/cultcargo/genesis/pfb-imaging/latest/fastim.yaml deleted file mode 100644 index f9b58bd..0000000 --- a/cultcargo/genesis/pfb-imaging/latest/fastim.yaml +++ /dev/null @@ -1,157 +0,0 @@ -inputs: - ms: - dtype: URI - required: true - abbreviation: ms - info: Path to measurement set - scans: - dtype: List[int] - info: List of SCAN_NUMBERS to image. Defaults to all. Input as string eg. '[0,2]' - if running from CLI - ddids: - dtype: List[int] - info: List of DATA_DESC_ID's to images. Defaults to all. Input as string eg. '[0,1]' - if running from CLI - fields: - dtype: List[int] - info: List of FIELD_ID's to image. Defaults to all. Input as string eg. '[0,1,2]' - if running from CLI - freq-range: - dtype: str - info: Frequency range to image in Hz. Specify as a string with colon delimiter - eg. '1e9:1.1e9' - overwrite: - dtype: bool - default: false - info: Allow overwrite of output xds - transfer-model-from: - dtype: str - abbreviation: tmf - info: Name of dataset to use for model initialisation - data-column: - dtype: str - default: DATA - abbreviation: dc - info: Data column to image. Must be the same across MSs - weight-column: - dtype: str - abbreviation: wc - info: Column containing natural weights. Must be the same across MSs - sigma-column: - dtype: str - abbreviation: sc - info: Column containing standard devations. Will be used to initialise natural - weights if detected. Must be the same across MSs - flag-column: - dtype: str - default: FLAG - abbreviation: fc - info: Column containing data flags. Must be the same across MSs - gain-table: - dtype: URI - info: Path to Quartical gain table containing NET gains. - integrations-per-image: - dtype: int - abbreviation: ipi - default: -1 - info: Number of time integrations per image. Default (-1, 0, None) -> dataset - per scan. - channels-per-degrid-image: - dtype: int - abbreviation: cpdi - default: 32 - info: Number of channels per image for degridding resolution. Any of (-1, 0, None) - implies single dataset per spw. - channels-per-grid-image: - dtype: int - abbreviation: cpgi - default: -1 - info: Number of channels per image for gridding resolution. Default of (-1, 0, - None) implies a single dataset per spw. - precision: - dtype: str - default: double - choices: - - single - - double - info: Gridding precision - field-of-view: - dtype: float - abbreviation: fov - default: 3.0 - info: Field of view in degrees - super-resolution-factor: - dtype: float - default: 2 - abbreviation: srf - info: Will over-sample Nyquist by this factor at max frequency - cell-size: - dtype: float - info: Cell size in arc-seconds - nx: - dtype: int - info: Number of x pixels - ny: - dtype: int - info: Number of y pixels - robustness: - dtype: float - abbreviation: rob - info: Robustness factor for Briggs weighting. None means natural - filter-extreme-counts: - dtype: bool - default: false - info: Replace extreme outliers in the weighting grid by their local mean - filter-nbox: - dtype: int - default: 16 - info: The size of the box to compute local mean over - filter-level: - dtype: float - default: 10 - info: The level above local mean that renders a grid weight extreme - target: - dtype: str - info: This can be predefined celestial objects known to astropy or a string in - the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - l2reweight-dof: - dtype: float - info: The degrees of freedom parameter for L2 reweighting. The default (None) - means no reweighting. A sensible value for this parameter depends on the level - of RFI in the data. Small values (eg. 2) result in aggressive reweighting and - should be avoided if the model is still incomplete. - host-address: - dtype: str - abbreviation: ha - info: Address where the distributed client lives. Uses LocalCluster if no address - is provided and scheduler is set to distributed. - nworkers: - dtype: int - default: 1 - abbreviation: nw - info: Number of worker processes. Use with distributed scheduler. - nthreads-dask: - dtype: int - default: 1 - abbreviation: ntd - info: Number of dask threads. This will largely determine the memory footprint. - Used as nthreads per worker when using distributed scheduler. - nvthreads: - dtype: int - abbreviation: nvt - info: Number of threads used to scale vertically (eg. for FFTs and gridding). - Each dask thread can in principle spawn this many threads. Will attempt to use - half the available threads by default. - progressbar: - dtype: bool - default: true - info: Display progress. Use --no-progressbar to deactivate. - output-format: - default: zarr - info: zarr or fits output - - _include: - - (.)gridding.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml b/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml deleted file mode 100644 index fe18f62..0000000 --- a/cultcargo/genesis/pfb-imaging/latest/fwdbwd.yaml +++ /dev/null @@ -1,95 +0,0 @@ -inputs: - _include: - - (.)gridding.yml - - (.)pdopts.yml - - (.)pmopts.yml - - (.)cgopts.yml - - (.)dist.yml - - (.)out.yml - model-name: - dtype: str - abbreviation: mname - default: MODEL - info: Name of model in mds - mask: - dtype: str - abbreviation: mask - info: Either path to mask.fits or set to mds to use the mask contained in the - mds. - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - suffix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data products - sigmainv: - dtype: float - default: 1e-5 - abbreviation: sinv - info: Standard deviation of assumed GRF prior - bases: - dtype: str - default: self,db1,db2 - abbreviation: bases - info: Wavelet bases to use. Give as comma separated str eg. 'self,db1,db2' - nlevels: - dtype: int - default: 3 - abbreviation: nlevels - info: Wavelet decomposition level - sigma21: - dtype: float - abbreviation: sig21 - info: Wavelet thresholding level - l1-reweight-from: - dtype: int - default: 5 - info: Start doing l1reweights after this many iterations - rmsfactor: - dtype: float - default: 3.0 - info: By default will threshold by rmsfactor*rms at every iteration - gamma: - dtype: float - default: 0.5 - info: Step size of update - positivity: - dtype: int - default: 1 - # choices: - # - 0 - # - 1 - # - 2 - info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity - constraint 2 -> strong positivity i.e. all pixels in a band > 0 - niter: - dtype: int - default: 5 - info: Number of iterations. L21 reweighting will take place after every iteration - abbreviation: niter - tol: - dtype: float - default: 1e-5 - info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) - < tol - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set - parametrisation: - dtype: str - default: id - info: The kind of parametrisation to apply - restart: - dtype: bool - default: false - info: Restart the deconvolution by initialising the model to zero and the residual - to the dirty image - -outputs: {} - -policies: - pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/imit.yaml b/cultcargo/genesis/pfb-imaging/latest/imit.yaml deleted file mode 100644 index d7c8a3d..0000000 --- a/cultcargo/genesis/pfb-imaging/latest/imit.yaml +++ /dev/null @@ -1,97 +0,0 @@ -inputs: - ms: - dtype: List[URI] - required: true - abbreviation: ms - info: Path to measurement set - scans: - dtype: List[int] - info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated - list 0,2 if running from CLI - ddids: - dtype: List[int] - info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated - list 0,2 if running from CLI - fields: - dtype: List[int] - info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list - 0,2 if running from CLI - freq-range: - dtype: str - info: Frequency range to image in Hz. Specify as a string with colon delimiter - eg. '1e9:1.1e9' - overwrite: - dtype: bool - default: false - info: Allow overwrite of output xds - radec: - dtype: Optional[List[any]] - info: Rephase all images to this radec specified in radians - data-column: - dtype: str - default: DATA - abbreviation: dc - info: Data column to image. Must be the same across MSs - weight-column: - dtype: str - abbreviation: wc - info: Column containing natural weights. Must be the same across MSs - sigma-column: - dtype: str - abbreviation: sc - info: Column containing standard devations. Will be used to initialise natural - weights if detected. Must be the same across MSs - flag-column: - dtype: str - default: FLAG - abbreviation: fc - info: Column containing data flags. Must be the same across MSs - gain-table: - dtype: List[str] - info: Path to Quartical gain table containing NET gains. There must be a table - for each MS and glob(ms) and glob(gt) should match up when running from CLI. - integrations-per-image: - dtype: int - abbreviation: ipi - default: -1 - info: Number of time integrations per image. Default (-1, 0, None) -> dataset - per scan. - channels-per-image: - dtype: int - abbreviation: cpi - default: 32 - info: Number of channels per image for degridding resolution. Any of (-1, 0, None) - implies single dataset per spw. - precision: - dtype: str - default: double - choices: - - single - - double - info: Gridding precision - bda-decorr: - dtype: float - default: 1.0 - info: BDA decorrelation factor. Only has an effect if less than one - beam-model: - dtype: str - info: Beam model to use - chan-average: - dtype: int - default: 1 - info: Average this number if channels together - target: - dtype: str - info: This can be predefined celestial objects known to astropy or a string in - the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - progressbar: - dtype: bool - default: true - info: Display progress. Use --no-progressbar to deactivate. - - - _include: - - (.)dist.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/pfb-imaging.yml b/cultcargo/pfb-imaging.yml index d2bb096..6cfeb35 100644 --- a/cultcargo/pfb-imaging.yml +++ b/cultcargo/pfb-imaging.yml @@ -16,10 +16,6 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image - pfb.imit: - image: - _use: vars.cult-cargo.pfb-imaging.image - pfb.grid: image: _use: vars.cult-cargo.pfb-imaging.image @@ -36,7 +32,7 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image - pfb.fwdbwd: + pfb.fluxmop: image: _use: vars.cult-cargo.pfb-imaging.image @@ -44,6 +40,14 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image + pfb.sara: + image: + _use: vars.cult-cargo.pfb-imaging.image + pfb.model2comps: image: _use: vars.cult-cargo.pfb-imaging.image + + pfb.hci: + image: + _use: vars.cult-cargo.pfb-imaging.image From 0c05104ac5dc10e10bb285f6af83865a64a653af Mon Sep 17 00:00:00 2001 From: landmanbester Date: Wed, 18 Sep 2024 14:42:04 +0200 Subject: [PATCH 12/21] update pfb config files --- cultcargo/genesis/pfb-imaging/latest/degrid.yaml | 5 +++++ cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml | 2 +- cultcargo/genesis/pfb-imaging/latest/hci.yaml | 2 +- cultcargo/genesis/pfb-imaging/latest/sara.yaml | 7 ++++--- cultcargo/genesis/pfb-imaging/latest/spotless.yaml | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml index 45e8adc..bb8571b 100644 --- a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml @@ -38,6 +38,11 @@ inputs: dtype: bool default: false info: Accumulate onto model column + region-file: + dtype: str + info: A region file containing regions that need to be converted to separate measurement + set columns. Each region in the file will end up in a separate column labelled + as model-column{#} with the remainder going into model-column. _include: - (.)gridding.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml index 8cfe0c5..ce8d8ca 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml @@ -22,7 +22,7 @@ inputs: dtype: float default: 1e-5 info: If using mask to construct model construct it where model > min-model - sigmainvsq: + eta: dtype: float default: 1e-5 abbreviation: sinv diff --git a/cultcargo/genesis/pfb-imaging/latest/hci.yaml b/cultcargo/genesis/pfb-imaging/latest/hci.yaml index b679310..6d313ba 100644 --- a/cultcargo/genesis/pfb-imaging/latest/hci.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/hci.yaml @@ -117,7 +117,7 @@ inputs: output-format: default: zarr info: zarr or fits output - sigmainvsq: + eta: dtype: float default: 1e-5 info: Value to add to Hessian to make it invertible. Smaller values tend to fit diff --git a/cultcargo/genesis/pfb-imaging/latest/sara.yaml b/cultcargo/genesis/pfb-imaging/latest/sara.yaml index 5e948c0..31010ff 100644 --- a/cultcargo/genesis/pfb-imaging/latest/sara.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/sara.yaml @@ -3,6 +3,7 @@ inputs: - (.)gridding.yml - (.)pdopts.yml - (.)pmopts.yml + - (.)cgopts.yml - (.)out.yml suffix: dtype: str @@ -29,7 +30,7 @@ inputs: info: Spectral norm of hessian approximation hess-approx: dtype: str - default: direct + default: psf abbreviation: hess-approx choices: [wgt, psf, direct] info: Which Hessian approximation to use. Set to wgt for vis space approximation, @@ -38,10 +39,10 @@ inputs: dtype: float default: 1.0 info: By default will threshold by rmsfactor*rms at every iteration - epsfactor: + eta: dtype: float default: 1.0 - info: Will use epsfactor*wsum to regularise the inversion of the Hessian approximation. + info: Will use eta*wsum to regularise the inversion of the Hessian approximation. gamma: dtype: float default: 1.0 diff --git a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml index 945fe88..b5dd1d5 100644 --- a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml @@ -132,7 +132,7 @@ inputs: dtype: bool default: false info: Hard reset the cache - sigmainvsq: + eta: dtype: float default: 1.0 info: Value to add to Hessian to make it invertible From 683fdaa785f97ef53a284ad4d53504799145b1a2 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Fri, 25 Oct 2024 10:48:43 +0200 Subject: [PATCH 13/21] remove default multiscale-max-scales from wsclean cab --- cultcargo/genesis/wsclean/wsclean-base.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/cultcargo/genesis/wsclean/wsclean-base.yml b/cultcargo/genesis/wsclean/wsclean-base.yml index c4479d8..b29fc9e 100644 --- a/cultcargo/genesis/wsclean/wsclean-base.yml +++ b/cultcargo/genesis/wsclean/wsclean-base.yml @@ -72,7 +72,6 @@ lib: repeat: ',' multiscale-max-scales: dtype: int - default: 8 multiscale-gain: dtype: float default: 0.15 From 9d9144a399c04c35f5bd7c2ed5543a4f002b440d Mon Sep 17 00:00:00 2001 From: landmanbester Date: Sat, 30 Nov 2024 10:31:59 +0200 Subject: [PATCH 14/21] update QuartiCal config --- cultcargo/genesis/quartical/argument_schema.yaml | 6 ++++++ cultcargo/genesis/quartical/gain_schema.yaml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/cultcargo/genesis/quartical/argument_schema.yaml b/cultcargo/genesis/quartical/argument_schema.yaml index d3a54a9..af3b369 100644 --- a/cultcargo/genesis/quartical/argument_schema.yaml +++ b/cultcargo/genesis/quartical/argument_schema.yaml @@ -423,3 +423,9 @@ dask: info: Which dask scheduler to use. The default, threads, is the most appropriate for non-cluster use. + + scheduler_plugin: + dtype: bool + default: true + info: + Enable or disable the dask scheduler plugin. diff --git a/cultcargo/genesis/quartical/gain_schema.yaml b/cultcargo/genesis/quartical/gain_schema.yaml index 0a462bb..266c23b 100644 --- a/cultcargo/genesis/quartical/gain_schema.yaml +++ b/cultcargo/genesis/quartical/gain_schema.yaml @@ -8,12 +8,15 @@ gain: - amplitude - delay - delay_and_offset + - delay_and_tec - phase - tec_and_offset - rotation_measure - rotation - crosshand_phase + - crosshand_phase_null_v - leakage + - parallactic_angle info: Type of gain to solve for. From 89c674e1025735a7ef009137e24a737ddf0d55d0 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Sat, 30 Nov 2024 10:53:46 +0200 Subject: [PATCH 15/21] add quartical v0.2.4 to manifest and use python3.10 as base python image --- cultcargo/builder/cargo-manifest.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index 725431a..85f815e 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -148,9 +148,18 @@ images: package: git+https://github.com/SpheMakh/msutils quartical: + assign: + # base image for generic Python-based packages + base_python_image: python-astro:3.10 + # corresponding python binary + python: python3.10 versions: '0.2.2': package: quartical==0.2.2 + '0.2.3': + package: quartical==0.2.3 + '0.2.4': + package: quartical==0.2.4 latest: package: quartical master: @@ -166,6 +175,11 @@ images: package: git+https://github.com/ratt-ru/CubiCal pfb-imaging: + assign: + # base image for generic Python-based packages + base_python_image: python-astro:3.10 + # corresponding python binary + python: python3.10 versions: latest: package: git+https://github.com/ratt-ru/pfb-imaging@outputs From 56fd136b64094a82d6fe594711a7ed6fb8034d49 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Sat, 30 Nov 2024 17:12:44 +0200 Subject: [PATCH 16/21] sync pfb configs --- cultcargo/builder/cargo-manifest.yml | 2 +- .../genesis/pfb-imaging/latest/degrid.yaml | 12 ++- .../genesis/pfb-imaging/latest/grid.yaml | 12 ++- cultcargo/genesis/pfb-imaging/latest/hci.yaml | 12 ++- .../genesis/pfb-imaging/latest/init.yaml | 19 +++-- .../pfb-imaging/latest/model2comps.yaml | 11 ++- .../genesis/pfb-imaging/latest/sara.yaml | 16 +++- .../genesis/pfb-imaging/latest/smoovie.yaml | 82 +++---------------- .../pfb-imaging/latest/uncabbedcabs.yml | 21 +++-- cultcargo/genesis/pfb-imaging/sync_config.py | 2 +- 10 files changed, 93 insertions(+), 96 deletions(-) diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index 85f815e..2c7f7ee 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -182,7 +182,7 @@ images: python: python3.10 versions: latest: - package: git+https://github.com/ratt-ru/pfb-imaging@outputs + package: git+https://github.com/ratt-ru/pfb-imaging@v0.0.5-dev ddfacet: assign: diff --git a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml index bb8571b..1094b71 100644 --- a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml @@ -4,11 +4,15 @@ inputs: required: true abbreviation: ms info: Path to measurement set. + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products mds: dtype: str - required: true abbreviation: mds - info: Path to the mds that needs to be degridded + info: Optional path to mds to use for degridding. By default mds is inferred from + output-filename. model-column: dtype: str default: MODEL_DATA @@ -32,8 +36,8 @@ inputs: channels-per-image: dtype: int abbreviation: cpi - default: -1 - info: Number of channels per image. Default (-1, 0, None) -> dataset per spw. + info: Number of channels per image. Default (None) -> read mapping from dds. (-1, + 0) -> one band per SPW. accumulate: dtype: bool default: false diff --git a/cultcargo/genesis/pfb-imaging/latest/grid.yaml b/cultcargo/genesis/pfb-imaging/latest/grid.yaml index c217f0a..e76981e 100644 --- a/cultcargo/genesis/pfb-imaging/latest/grid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/grid.yaml @@ -43,9 +43,13 @@ inputs: dtype: bool default: true info: Compute noise map by sampling from weights + beam: + dtype: bool + default: true + info: Interpolate average beam pattern psf-oversize: dtype: float - default: 1.0 + default: 1.4 abbreviation: psfo info: Size of PSF relative to dirty image weight: @@ -92,7 +96,11 @@ inputs: - (.)dist.yml - (.)out.yml -outputs: {} +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/hci.yaml b/cultcargo/genesis/pfb-imaging/latest/hci.yaml index 6d313ba..58f1000 100644 --- a/cultcargo/genesis/pfb-imaging/latest/hci.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/hci.yaml @@ -134,14 +134,22 @@ inputs: dtype: bool default: false info: Report worker memory as tasks complete - + check-ants: + dtype: bool + default: false + info: Check that ANTENNA1 and ANTENNA2 tables are consistent with the ANTENNA + table. _include: - (.)gridding.yml - (.)out.yml - (.)dist.yml - (.)cgopts.yml -outputs: {} +outputs: + dir-out: + implicit: '{current.output-filename}_{current.product}.fds' + dtype: Directory + must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/init.yaml b/cultcargo/genesis/pfb-imaging/latest/init.yaml index b9f5759..f5dc3f0 100644 --- a/cultcargo/genesis/pfb-imaging/latest/init.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/init.yaml @@ -5,15 +5,15 @@ inputs: abbreviation: ms info: Path to measurement set scans: - dtype: Optional[List[int]] + dtype: List[int] info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated list 0,2 if running from CLI ddids: - dtype: Optional[List[int]] + dtype: List[int] info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated list 0,2 if running from CLI fields: - dtype: Optional[List[int]] + dtype: List[int] info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list 0,2 if running from CLI freq-range: @@ -96,16 +96,25 @@ inputs: dtype: bool default: true info: Display progress. Use --no-progressbar to deactivate. - memory_reporting: + memory-reporting: dtype: bool default: false info: Report worker memory as tasks complete + check-ants: + dtype: bool + default: false + info: Check that ANTENNA1 and ANTENNA2 tables are consistent with the ANTENNA + table. _include: - (.)out.yml - (.)dist.yml -outputs: {} +outputs: + xds-out: + implicit: '{current.output-filename}_{current.product}.xds' + dtype: Directory + must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml index fe8229a..fd88c1d 100644 --- a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml @@ -62,7 +62,16 @@ inputs: _include: - (.)out.yml -outputs: {} +outputs: + mds-out: + implicit: =IFSET(current.model-out, current.model-out, + '{current.output-filename}_{current.product}_{current.suffix}_{current.model-name}.mds') + dtype: Directory + must_exist: false + # fits-out: + # implicit: =IFSET(current.model-out, STRIPEXT(current.model-out)+.fits, '{current.output-filename}_{current.product}_{current.suffix}_{current.model-name}.mds') + # dtype: Directory + # must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/sara.yaml b/cultcargo/genesis/pfb-imaging/latest/sara.yaml index 31010ff..d277d9b 100644 --- a/cultcargo/genesis/pfb-imaging/latest/sara.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/sara.yaml @@ -64,7 +64,7 @@ inputs: # - 1 # - 2 info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity - constraint and 2 -> strong positivity i.e. all pixels in a band > 0 + constraint 2 -> strong positivity i.e. all pixels in a band > 0 niter: dtype: int default: 10 @@ -97,8 +97,20 @@ inputs: dtype: float default: 0.5 info: Reduce the regularisation strength by this fraction at the outset. + verbosity: + dtype: int + default: 1 + info: Set to larger than 1 to report timings during residual computation -outputs: {} +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false + mds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}_model.mds' + dtype: Directory + must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml index 4ef4178..955bdf0 100644 --- a/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml @@ -1,13 +1,8 @@ inputs: fds: dtype: str - required: true - info: Path to fds produced by fastim worker - outname: - dtype: str - required: true - info: The name of the output. Fits images will be suffixed by band and time ID's. - Mp4 movies will suffixed by .mp4 + info: Optional path to fds produced by fastim worker. Default naming convention + is used if not provided. animate-axis: dtype: str default: time @@ -29,25 +24,12 @@ inputs: info: vmax will be set as min-frac * median(rms) cmap: dtype: str - default: Grays + default: gray info: Any valid matplotlib cmap text-colour: dtype: str default: cyan info: Colour to render text on image - log-directory: - dtype: str - required: true - abbreviation: ldir - info: Directory to write logs and performance reports to - freq-bin: - dtype: int - default: 1 - info: How many freqs to bin into single frame - time-bin: - dtype: int - default: 1 - info: How many times to bin into single frame freq-range: dtype: str info: Frequency range to consider. Specify as a string with colon delimiter eg. @@ -60,57 +42,6 @@ inputs: dtype: bool default: false info: Allow overwrite of output files with the same names. - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - movie-mfs: - dtype: bool - default: true - info: Output MFS movie - movie-cubes: - dtype: bool - default: false - info: Output movie per freq band - host-address: - dtype: str - abbreviation: ha - info: Address where the distributed client lives. Uses LocalCluster if no address - is provided and scheduler is set to distributed. - nworkers: - dtype: int - default: 1 - abbreviation: nw - info: Number of worker processes. Use with distributed scheduler. - nthreads: - dtype: int - default: 1 - abbreviation: nt - info: Number of threads. - # TBD - field-of-view: - dtype: float - abbreviation: fov - default: 3.0 - info: Field of view in degrees - nx: - dtype: int - info: Number of x pixels - ny: - dtype: int - info: Number of y pixels - target: - dtype: str - info: This can be predefined celestial objects known to astropy or a string in - the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - respect-scan-boundaries: - dtype: bool - default: true - info: Whether to allow averaging across scan boundaries out-format: dtype: str default: gif @@ -127,6 +58,13 @@ inputs: dtype: int default: 12 info: Constant rate factor for controlling mp4 output quality. + scratch-dir: + dtype: URI + info: Streamjoy scratch directory. + + _include: + - (.)out.yml + - (.)dist.yml outputs: {} diff --git a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml index 91590c2..f7e23d3 100644 --- a/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml +++ b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml @@ -25,14 +25,14 @@ pfb.degrid: _include: - (.)degrid.yaml -pfb.klean: - command: pfb.workers.klean.klean +pfb.kclean: + command: pfb.workers.kclean.kclean flavour: python policies: pass_missing_as_none: true _include: - - (.)klean.yaml + - (.)kclean.yaml pfb.restore: command: pfb.workers.restore.restore @@ -43,14 +43,14 @@ pfb.restore: _include: - (.)restore.yaml -pfb.fluxmop: - command: pfb.workers.fluxmop.fluxmop +pfb.fluxtractor: + command: pfb.workers.fluxtractor.fluxtractor flavour: python policies: pass_missing_as_none: true _include: - - (.)fluxmop.yaml + - (.)fluxtractor.yaml pfb.spotless: command: pfb.workers.spotless.spotless @@ -87,3 +87,12 @@ pfb.hci: _include: - (.)hci.yaml + +pfb.smoovie: + command: pfb.workers.smoovie.smoovie + flavour: python + policies: + pass_missing_as_none: true + + _include: + - (.)smoovie.yaml diff --git a/cultcargo/genesis/pfb-imaging/sync_config.py b/cultcargo/genesis/pfb-imaging/sync_config.py index 5bfb478..1f60e4b 100644 --- a/cultcargo/genesis/pfb-imaging/sync_config.py +++ b/cultcargo/genesis/pfb-imaging/sync_config.py @@ -7,7 +7,7 @@ mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-imaging/latest' -branch = 'outputs' +branch = 'v0.0.5-dev' content_path = f'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/{branch}/pfb/parser' # returns list of files in subfolder From 44ed8e355210fa421b4f10548f294780aa643d13 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Sun, 1 Dec 2024 09:52:24 +0200 Subject: [PATCH 17/21] fix pfb configs --- .../latest/{fluxmop.yaml => fluxtractor.yaml} | 12 ++++++------ .../pfb-imaging/latest/{klean.yaml => kclean.yaml} | 8 ++++++-- cultcargo/pfb-imaging.yml | 8 ++++++-- 3 files changed, 18 insertions(+), 10 deletions(-) rename cultcargo/genesis/pfb-imaging/latest/{fluxmop.yaml => fluxtractor.yaml} (90%) rename cultcargo/genesis/pfb-imaging/latest/{klean.yaml => kclean.yaml} (94%) diff --git a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml b/cultcargo/genesis/pfb-imaging/latest/fluxtractor.yaml similarity index 90% rename from cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml rename to cultcargo/genesis/pfb-imaging/latest/fluxtractor.yaml index ce8d8ca..44628b4 100644 --- a/cultcargo/genesis/pfb-imaging/latest/fluxmop.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/fluxtractor.yaml @@ -25,12 +25,8 @@ inputs: eta: dtype: float default: 1e-5 - abbreviation: sinv + abbreviation: eta info: Standard deviation of assumed GRF prior - sigma: - dtype: float - default: 1 - info: The value that is added to |psfhat| for preconditioning. model-name: dtype: str default: MODEL @@ -57,7 +53,11 @@ inputs: - (.)dist.yml - (.)out.yml -outputs: {} +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/klean.yaml b/cultcargo/genesis/pfb-imaging/latest/kclean.yaml similarity index 94% rename from cultcargo/genesis/pfb-imaging/latest/klean.yaml rename to cultcargo/genesis/pfb-imaging/latest/kclean.yaml index fe0869f..4d1f703 100644 --- a/cultcargo/genesis/pfb-imaging/latest/klean.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/kclean.yaml @@ -34,7 +34,7 @@ inputs: info: Absolute threshold at which to stop cleaning. By default it is set automatically using sigmathreshold parameter abbreviation: th - sigmathreshold: + rmsfactor: dtype: float default: 3.0 info: Multiple of the rms at which to terminate the minor cycle @@ -108,7 +108,11 @@ inputs: - (.)gridding.yml - (.)out.yml -outputs: {} +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false policies: pass_missing_as_none: true diff --git a/cultcargo/pfb-imaging.yml b/cultcargo/pfb-imaging.yml index 6cfeb35..271bcd4 100644 --- a/cultcargo/pfb-imaging.yml +++ b/cultcargo/pfb-imaging.yml @@ -24,7 +24,7 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image - pfb.klean: + pfb.kclean: image: _use: vars.cult-cargo.pfb-imaging.image @@ -32,7 +32,7 @@ cabs: image: _use: vars.cult-cargo.pfb-imaging.image - pfb.fluxmop: + pfb.fluxtractor: image: _use: vars.cult-cargo.pfb-imaging.image @@ -51,3 +51,7 @@ cabs: pfb.hci: image: _use: vars.cult-cargo.pfb-imaging.image + + pfb.smoovie: + image: + _use: vars.cult-cargo.pfb-imaging.image From 15f04e65f01340fd6803fbf07d3cc0f330bfb822 Mon Sep 17 00:00:00 2001 From: Jonathan Kenyon Date: Mon, 2 Dec 2024 18:02:28 +0200 Subject: [PATCH 18/21] Bump stimela. New QC image. --- cultcargo/builder/cargo-manifest.yml | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index 2c7f7ee..d692edc 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -160,6 +160,8 @@ images: package: quartical==0.2.3 '0.2.4': package: quartical==0.2.4 + '0.2.5': + package: quartical==0.2.5 latest: package: quartical master: diff --git a/pyproject.toml b/pyproject.toml index c830df3..bd82dd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ packages = [{include = "cultcargo"}] [tool.poetry.dependencies] python = "^3.8" -stimela = "^2.0" +stimela = "^2.0.1.1" requests = "^2.0" [tool.poetry.scripts] From 470f58e3c650a9ac892cb298a3794ca244b87016 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Tue, 10 Dec 2024 14:59:11 +0200 Subject: [PATCH 19/21] use v0.0.5-dev for latest version --- cultcargo/builder/cargo-manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index 592f934..4b252a8 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -215,7 +215,7 @@ images: '0.0.4': package: pfb-imaging==0.0.4 latest: - package: git+https://github.com/ratt-ru/pfb-imaging@pfb-imaging + package: git+https://github.com/ratt-ru/pfb-imaging@v0.0.5-dev master: package: git+https://github.com/ratt-ru/pfb-imaging From 33b3d7fde77fd82628d45e76ed98cee1a711d301 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Fri, 13 Dec 2024 17:24:17 +0200 Subject: [PATCH 20/21] add pfb v0.0.5 --- cultcargo/builder/cargo-manifest.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index 4b252a8..605d6b8 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -211,11 +211,18 @@ images: package: git+https://github.com/ratt-ru/CubiCal pfb-imaging: + assign: + # base image for generic Python-based packages + base_python_image: python-astro:3.10 + # corresponding python binary + python: python3.10 versions: '0.0.4': package: pfb-imaging==0.0.4 + '0.0.5': + package: pfb-imaging==0.0.4 latest: - package: git+https://github.com/ratt-ru/pfb-imaging@v0.0.5-dev + package: git+https://github.com/ratt-ru/pfb-imaging master: package: git+https://github.com/ratt-ru/pfb-imaging From 1d95887ecf5e099b9a8fee385405cbb379fc9df0 Mon Sep 17 00:00:00 2001 From: landmanbester Date: Fri, 13 Dec 2024 17:33:25 +0200 Subject: [PATCH 21/21] use main branch to sync pfb config --- cultcargo/genesis/pfb-imaging/sync_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cultcargo/genesis/pfb-imaging/sync_config.py b/cultcargo/genesis/pfb-imaging/sync_config.py index 1f60e4b..04f18f0 100644 --- a/cultcargo/genesis/pfb-imaging/sync_config.py +++ b/cultcargo/genesis/pfb-imaging/sync_config.py @@ -7,7 +7,7 @@ mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-imaging/latest' -branch = 'v0.0.5-dev' +branch = 'main' content_path = f'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/{branch}/pfb/parser' # returns list of files in subfolder