diff --git a/cultcargo/builder/cargo-manifest.yml b/cultcargo/builder/cargo-manifest.yml index 7730905..a03811a 100644 --- a/cultcargo/builder/cargo-manifest.yml +++ b/cultcargo/builder/cargo-manifest.yml @@ -187,17 +187,24 @@ images: package: git+https://github.com/SpheMakh/msutils quartical: + assign: + # base image for generic Python-based packages + base_python_image: python-astro:3.10 + # corresponding python binary + python: python3.10 versions: - master: - package: git+https://github.com/ratt-ru/QuartiCal '0.2.2': package: quartical==0.2.2 '0.2.3': package: quartical==0.2.3 '0.2.4': package: quartical==0.2.4 + '0.2.5': + package: quartical==0.2.5 latest: package: quartical + master: + package: git+https://github.com/ratt-ru/QuartiCal cubical: versions: @@ -209,11 +216,20 @@ images: package: git+https://github.com/ratt-ru/CubiCal pfb-imaging: + assign: + # base image for generic Python-based packages + base_python_image: python-astro:3.10 + # corresponding python binary + python: python3.10 versions: '0.0.4': package: pfb-imaging==0.0.4 - # master: - # package: git+https://github.com/ratt-ru/pfb-imaging + '0.0.5': + package: pfb-imaging==0.0.4 + latest: + package: git+https://github.com/ratt-ru/pfb-imaging + master: + package: git+https://github.com/ratt-ru/pfb-imaging ddfacet: assign: diff --git a/cultcargo/genesis/pfb-clean/latest/degrid.yaml b/cultcargo/genesis/pfb-clean/latest/degrid.yaml deleted file mode 100644 index 1023ddb..0000000 --- a/cultcargo/genesis/pfb-clean/latest/degrid.yaml +++ /dev/null @@ -1,42 +0,0 @@ -inputs: - ms: - dtype: URI - required: true - abbreviation: ms - info: Path to measurement set. - mds: - dtype: str - required: true - abbreviation: mds - info: Path to the mds that needs to be degridded - model-column: - dtype: str - default: MODEL_DATA - info: Column to write model data to - product: - dtype: str - abbreviation: p - default: I - info: Imaging products to degrid. Options are I, Q, U, V. Only single Stokes products - are currently supported. - integrations-per-image: - dtype: int - default: -1 - abbreviation: ipi - info: Number of time integrations corresponding to each image. Default -1 (equivalently - 0 or None) implies degrid per scan. - channels-per-image: - dtype: int - abbreviation: cpi - default: -1 - info: Number of channels per image. Default (-1, 0, None) -> dataset per spw. - accumulate: - dtype: bool - default: false - info: Accumulate onto model column - - _include: - - (.)gridding.yml - - (.)dist.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/fastim.yaml b/cultcargo/genesis/pfb-clean/latest/fastim.yaml deleted file mode 100644 index 6248b60..0000000 --- a/cultcargo/genesis/pfb-clean/latest/fastim.yaml +++ /dev/null @@ -1,96 +0,0 @@ -inputs: - ms: - dtype: List[str] - required: true - abbreviation: ms - info: Path to measurement set - scans: - dtype: List[int] - info: List of SCAN_NUMBERS to image. Defaults to all. Input as string eg. '[0,2]' - if running from CLI - ddids: - dtype: List[int] - info: List of DATA_DESC_ID's to images. Defaults to all. Input as string eg. '[0,1]' - if running from CLI - fields: - dtype: List[int] - info: List of FIELD_ID's to image. Defaults to all. Input as string eg. '[0,1,2]' - if running from CLI - freq-range: - dtype: str - info: Frequency range to image in Hz. Specify as a string with colon delimiter - eg. '1e9:1.1e9' - overwrite: - dtype: bool - default: false - info: Allow overwrite of output xds - radec: - dtype: Optional[List[any]] - info: Rephase all images to this radec specified in radians - data-column: - dtype: str - default: DATA - abbreviation: dc - info: Data column to image. Must be the same across MSs - weight-column: - dtype: str - abbreviation: wc - info: Column containing natural weights. Must be the same across MSs - sigma-column: - dtype: str - abbreviation: sc - info: Column containing standard devations. Will be used to initialise natural - weights if detected. Must be the same across MSs - flag-column: - dtype: str - default: FLAG - abbreviation: fc - info: Column containing data flags. Must be the same across MSs - gain-table: - dtype: List[str] - info: Path to Quartical gain table containing NET gains. There must be a table - for each MS and glob(ms) and glob(gt) should match up - gain-term: - dtype: str - default: NET - info: Which gain term to use. Default is NET - integrations-per-image: - dtype: int - abbreviation: ipi - default: -1 - info: Number of time integrations per image. Default (-1, 0, None) -> dataset - per scan. - channels-per-image: - dtype: int - abbreviation: cpi - default: -1 - info: Number of channels per image. Default (-1, 0, None) -> dataset per spw. - precision: - dtype: str - default: double - choices: - - single - - double - info: Gridding precision - bda-decorr: - dtype: float - default: 1.0 - info: BDA decorrelation factor. Only has an effect if less than one - beam-model: - dtype: str - info: Beam model to use - max-field-of-view: - dtype: float - abbreviation: fov - default: 3.0 - info: Field of view in degrees - chan-average: - dtype: int - default: 1 - info: Average this number if channels together - - _include: - - (.)dist.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/forward.yaml b/cultcargo/genesis/pfb-clean/latest/forward.yaml deleted file mode 100644 index fbf816a..0000000 --- a/cultcargo/genesis/pfb-clean/latest/forward.yaml +++ /dev/null @@ -1,42 +0,0 @@ -inputs: - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data product - mask: - dtype: str - abbreviation: mask - info: Either path to mask.fits or set to model to construct from model - sigmainv: - dtype: float - default: 1e-5 - abbreviation: sinv - info: Standard deviation of assumed GRF prior - gamma: - dtype: float - default: 1.0 - info: Step size of update - use-psf: - dtype: bool - default: true - info: Whether to approximate the Hessian as a convolution by the PSF - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - _include: - - (.)gridding.yml - - (.)cgopts.yml - - (.)dist.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/fwdbwd.yaml b/cultcargo/genesis/pfb-clean/latest/fwdbwd.yaml deleted file mode 100644 index e1c54af..0000000 --- a/cultcargo/genesis/pfb-clean/latest/fwdbwd.yaml +++ /dev/null @@ -1,100 +0,0 @@ -inputs: - _include: - - (.)gridding.yml - - (.)pdopts.yml - - (.)pmopts.yml - - (.)cgopts.yml - - (.)dist.yml - - (.)out.yml - model-name: - dtype: str - abbreviation: mname - default: MODEL - info: Name of model in mds - mask: - dtype: str - abbreviation: mask - info: Either path to mask.fits or set to mds to use the mask contained in the - mds. - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data products - sigmainv: - dtype: float - default: 1e-5 - abbreviation: sinv - info: Standard deviation of assumed GRF prior - bases: - dtype: str - default: self,db1,db2 - abbreviation: bases - info: Wavelet bases to use. Give as comma separated str eg. 'self,db1,db2' - nlevels: - dtype: int - default: 3 - abbreviation: nlevels - info: Wavelet decomposition level - sigma21: - dtype: float - abbreviation: sig21 - info: Wavelet thresholding level - l1reweight-from: - dtype: int - default: 5 - info: Start doing l1reweights after this many iterations - rmsfactor: - dtype: float - default: 3.0 - info: By default will threshold by rmsfactor*rms at every iteration - gamma: - dtype: float - default: 0.5 - info: Step size of update - positivity: - dtype: int - default: 1 - # choices: - # - 0 - # - 1 - # - 2 - info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity - constraint 2 -> strong positivity i.e. all pixels in a band > 0 - niter: - dtype: int - default: 5 - info: Number of iterations. L21 reweighting will take place after every iteration - abbreviation: niter - tol: - dtype: float - default: 1e-5 - info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) - < tol - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set - parametrisation: - dtype: str - default: id - info: The kind of parametrisation to apply - restart: - dtype: bool - default: false - info: Restart the deconvolution by initialising the model to zero and the residual - to the dirty image - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/init_ims.yaml b/cultcargo/genesis/pfb-clean/latest/init_ims.yaml deleted file mode 100644 index d461637..0000000 --- a/cultcargo/genesis/pfb-clean/latest/init_ims.yaml +++ /dev/null @@ -1,102 +0,0 @@ -inputs: - ms: - dtype: str - required: true - info: 'Path to measurement set.' - metavar: MS - abbreviation: ms - output_filename: - dtype: str - required: true - info: 'Basename of output' - metavar: BASENAME - abbreviation: o - overwrite: - dtype: bool - default: false - info: Allow overwrite of output xds - nband: - dtype: int - required: true - abbreviation: nb - info: 'Number of imaging bands' - data_column: - dtype: str - default: DATA - abbreviation: dc - info: 'Data column to image. Must be the same across MSs' - metavar: COLUMN - weight_column: - dtype: str - default: WEIGHT_SPECTRUM - abbreviation: wc - info: 'Column containing natural weights. Must be the same across MSs' - metavar: COLUMN - sigma_column: - dtype: str - abbreviation: sc - info: 'Column containing standard devations. \ - Will be used to initialise natural weights if detected. \ - Must be the same across MSs' - metavar: COLUMN - flag_column: - dtype: str - default: FLAG - abbreviation: fc - info: 'Column containing data flags. Must be the same across MSs' - metavar: COLUMN - gain_table: - dtype: str - info: "Path to Quartical gain table containing NET gains.\ - There must be a table for each MS and glob(ms) and glob(gt) \ - should match up." - gain_term: - dtype: str - default: NET - info: "Which gain term to use. Default is NET" - product: - dtype: str - abbreviation: p - default: I - info: 'Imaging products to produce. Options are I, Q, U, V. \ - Only single Stokes products are currently supported' - utimes_per_chunk: - dtype: int - info: 'Unique times per row chunk' - abbreviation: utpc - default: -1 - group_by_field: - dtype: bool - default: True - info: "Split data by field" - group_by_ddid: - dtype: bool - default: True - info: "Split data by DDID" - group_by_scan: - dtype: bool - default: True - info: "Split data by scan" - precision: - dtype: str - default: double - info: '"single" or "double"' - bda_decorr: - dtype: float - default: 1.0 - info: "BDA decorrelation factor. Only has an effect if less than one" - beam_model: - dtype: str - info: "Beam model to use" - max_field_of_view: - dtype: float - abbreviation: fov - default: 3.0 - info: 'Field of view in degrees' - metavar: DEG - - _include: - - dist.yml - -outputs: - {} diff --git a/cultcargo/genesis/pfb-clean/latest/model2comps.yaml b/cultcargo/genesis/pfb-clean/latest/model2comps.yaml deleted file mode 100644 index aa3b872..0000000 --- a/cultcargo/genesis/pfb-clean/latest/model2comps.yaml +++ /dev/null @@ -1,44 +0,0 @@ -inputs: - overwrite: - dtype: bool - default: false - info: Allow overwrite of existing model - nbasisf: - dtype: int - default: 4 - info: Order of interpolating polynomial for frequency axis - nbasist: - dtype: int - info: Order of interpolating polynomial for frequency axis - fit-mode: - dtype: str - default: Legendre - min-val: - dtype: float - info: Only fit components above this flux level - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data products - model-name: - dtype: str - default: MODEL - abbreviation: mname - info: Name of model in mds - model-fits: - dtype: str - info: Specify to degrid from a fits file - use-wsum: - dtype: bool - default: true - info: Use wsum as weights during fit - sigmasq: - dtype: float - default: 1e-10 - info: Multiple of the identity to add to the hessian for stability - - _include: - - (.)dist.yml - - (.)out.yml - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/restore.yaml b/cultcargo/genesis/pfb-clean/latest/restore.yaml deleted file mode 100644 index e984a8d..0000000 --- a/cultcargo/genesis/pfb-clean/latest/restore.yaml +++ /dev/null @@ -1,34 +0,0 @@ -inputs: - _include: - - (.)dist.yml - - (.)out.yml - model-name: - dtype: str - abbreviation: mname - default: MODEL - info: Name of model in mds - residual-name: - dtype: str - abbreviation: rname - default: RESIDUAL - info: Name of residual in dds - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data \ products - outputs: - dtype: str - default: mMrRiI - info: Output products. (m)odel, (r)esidual, (i)mage, (c)lean beam. \ Captitals - correspond to cubes. - overwrite: - dtype: bool - default: false - info: Allow overwrite of output xds - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/spotless.yaml b/cultcargo/genesis/pfb-clean/latest/spotless.yaml deleted file mode 100644 index e07ba1a..0000000 --- a/cultcargo/genesis/pfb-clean/latest/spotless.yaml +++ /dev/null @@ -1,75 +0,0 @@ -inputs: - _include: - - (.)gridding.yml - - (.)pdopts.yml - - (.)pmopts.yml - - (.)dist.yml - - (.)out.yml - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: - dtype: str - default: main - info: Can be used to specify a custom name for the image space data products - bases: - dtype: str - info: 'Wavelet bases to use. Give as comma separated str eg. ' - default: self,db1,db2 - abbreviation: bases - nlevels: - dtype: int - info: Wavelet decomposition level - default: 3 - abbreviation: nlevels - l1reweight-from: - dtype: int - default: 5 - info: Start doing l1reweights after this many iterations - hessnorm: - dtype: float - abbreviation: hessnorm - info: Spectral norm of hessian approximation - rmsfactor: - dtype: float - default: 3.0 - info: By default will threshold by rmsfactor*rms at every iteration - gamma: - dtype: float - default: 1.0 - info: Step size of update - positivity: - dtype: int - default: 1 - # choices: - # - 0 - # - 1 - # - 2 - info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity - constraint and 2 -> strong positivity i.e. all pixels in a band > 0 - niter: - dtype: int - default: 5 - abbreviation: niter - info: Number of iterations. L21 reweighting will take place after every iteration - tol: - dtype: float - default: 1e-3 - info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) - < tol - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set - -outputs: {} diff --git a/cultcargo/genesis/pfb-clean/latest/cgopts.yml b/cultcargo/genesis/pfb-imaging/latest/cgopts.yml similarity index 97% rename from cultcargo/genesis/pfb-clean/latest/cgopts.yml rename to cultcargo/genesis/pfb-imaging/latest/cgopts.yml index eafdc63..3f3cc55 100644 --- a/cultcargo/genesis/pfb-clean/latest/cgopts.yml +++ b/cultcargo/genesis/pfb-imaging/latest/cgopts.yml @@ -26,5 +26,5 @@ cg-report-freq: abbreviation: cgrf backtrack: dtype: bool - default: true + default: false info: Ensure residual decreases at every iteration diff --git a/cultcargo/genesis/pfb-imaging/latest/degrid.yaml b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml new file mode 100644 index 0000000..1094b71 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/degrid.yaml @@ -0,0 +1,59 @@ +inputs: + ms: + dtype: List[URI] + required: true + abbreviation: ms + info: Path to measurement set. + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products + mds: + dtype: str + abbreviation: mds + info: Optional path to mds to use for degridding. By default mds is inferred from + output-filename. + model-column: + dtype: str + default: MODEL_DATA + info: Column to write model data to + product: + dtype: str + abbreviation: p + default: I + info: Imaging products to degrid. Options are I, Q, U, V. Only single Stokes products + are currently supported. + freq-range: + dtype: str + info: Frequency range to image in Hz. Specify as a string with colon delimiter + eg. '1e9:1.1e9' + integrations-per-image: + dtype: int + default: -1 + abbreviation: ipi + info: Number of time integrations corresponding to each image. Default -1 (equivalently + 0 or None) implies degrid per scan. + channels-per-image: + dtype: int + abbreviation: cpi + info: Number of channels per image. Default (None) -> read mapping from dds. (-1, + 0) -> one band per SPW. + accumulate: + dtype: bool + default: false + info: Accumulate onto model column + region-file: + dtype: str + info: A region file containing regions that need to be converted to separate measurement + set columns. Each region in the file will end up in a separate column labelled + as model-column{#} with the remainder going into model-column. + + _include: + - (.)gridding.yml + - (.)dist.yml + - (.)out.yml + +outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/dist.yml b/cultcargo/genesis/pfb-imaging/latest/dist.yml similarity index 57% rename from cultcargo/genesis/pfb-clean/latest/dist.yml rename to cultcargo/genesis/pfb-imaging/latest/dist.yml index 2739012..397b022 100644 --- a/cultcargo/genesis/pfb-clean/latest/dist.yml +++ b/cultcargo/genesis/pfb-imaging/latest/dist.yml @@ -8,21 +8,18 @@ nworkers: default: 1 abbreviation: nw info: Number of worker processes. Use with distributed scheduler. -nthreads-dask: +nthreads: dtype: int - default: 1 - abbreviation: ntd - info: Number of dask threads. This will largely determine the memory footprint. - Used as nthreads per worker when using distributed scheduler. -nvthreads: - dtype: int - abbreviation: nvt + abbreviation: nt info: Number of threads used to scale vertically (eg. for FFTs and gridding). Each dask thread can in principle spawn this many threads. Will attempt to use half the available threads by default. -scheduler: +direct_to_workers: + dtype: bool + default: true + info: Connect direct to workers i.e. bypass scheduler. Faster but then the dashboard + isn't very useful. +log-level: dtype: str - default: threads - choices: [threads, single-threaded, sync, distributed, processes] - abbreviation: scheduler - info: Which dask scheduler to use. + default: error + choices: [error, warning, info, debug] diff --git a/cultcargo/genesis/pfb-imaging/latest/fluxtractor.yaml b/cultcargo/genesis/pfb-imaging/latest/fluxtractor.yaml new file mode 100644 index 0000000..44628b4 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/fluxtractor.yaml @@ -0,0 +1,63 @@ +inputs: + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data product + mask: + dtype: str + abbreviation: mask + info: Either path to mask.fits or set to model to construct from model + zero-model-outside-mask: + dtype: bool + default: false + info: Make sure the input model is zero outside the mask. Only has an effect if + an external mask has been passed in. A major cycle will be triggered to recompute + the residual after zeroing. + or-mask-with-model: + dtype: bool + default: false + info: Make a new mask consisting of the union of input mask and where the MFS + model is larger than zero. + min-model: + dtype: float + default: 1e-5 + info: If using mask to construct model construct it where model > min-model + eta: + dtype: float + default: 1e-5 + abbreviation: eta + info: Standard deviation of assumed GRF prior + model-name: + dtype: str + default: MODEL + info: Name of the model to update + residual-name: + dtype: str + default: RESIDUAL + info: Name of the residual to use + gamma: + dtype: float + default: 1.0 + info: Step size of update + use-psf: + dtype: bool + default: true + info: Whether to approximate the Hessian as a convolution by the PSF + memory-greedy: + dtype: bool + default: false + info: Holds data in memory if set + _include: + - (.)gridding.yml + - (.)cgopts.yml + - (.)dist.yml + - (.)out.yml + +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/grid.yaml b/cultcargo/genesis/pfb-imaging/latest/grid.yaml similarity index 70% rename from cultcargo/genesis/pfb-clean/latest/grid.yaml rename to cultcargo/genesis/pfb-imaging/latest/grid.yaml index 7a4db83..e76981e 100644 --- a/cultcargo/genesis/pfb-clean/latest/grid.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/grid.yaml @@ -1,13 +1,7 @@ inputs: xds: - info: Optional explicit path to xds. Set using output-filename and postfix by - default. - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: + info: Optional explicit path to xds. Set using output-filename and suffix by default. + suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products @@ -18,24 +12,21 @@ inputs: overwrite: dtype: bool default: false - info: Allow overwriting of image space data products. Specify postfix to create + info: Allow overwriting of image space data products. Specify suffix to create a new data set. transfer-model-from: dtype: str abbreviation: tmf info: Name of dataset to use for model initialisation + use-best-model: + dtype: bool + default: false + info: If this flag is set MODEL_BEST will be used as the model unless transfer-model-from + is specified. By default MODEL will be used as the model. robustness: dtype: float abbreviation: rob info: Robustness factor for Briggs weighting. None means natural - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes dirty: dtype: bool default: true @@ -44,19 +35,27 @@ inputs: dtype: bool default: true info: Compute the PSF + residual: + dtype: bool + default: true + info: Compute the residual (only if model is present) + noise: + dtype: bool + default: true + info: Compute noise map by sampling from weights + beam: + dtype: bool + default: true + info: Interpolate average beam pattern psf-oversize: dtype: float - default: 2.0 + default: 1.4 abbreviation: psfo info: Size of PSF relative to dirty image weight: dtype: bool default: true info: Compute effectve image space weights - residual: - dtype: bool - default: false - info: Compute the residual image field-of-view: dtype: float abbreviation: fov @@ -76,24 +75,18 @@ inputs: ny: dtype: int info: Number of y pixels - filter-extreme-counts: - dtype: bool - default: false - info: Replace extreme outliers in the weighting grid by their local mean - filter-nbox: - dtype: int - default: 16 - info: The size of the box to compute local mean over - filter-level: + filter-counts-level: dtype: float - default: 10 - info: The level above local mean that renders a grid weight extreme + default: 5.0 + info: Set minimum counts in the uniform weighting grid to the median divided by + this value. This is useful to avoid artificially up-weighting nearly empty uv-cells. target: dtype: str info: This can be predefined celestial objects known to astropy or a string in the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) - l2reweight-dof: + l2-reweight-dof: dtype: float + abbreviation: dof info: The degrees of freedom parameter for L2 reweighting. The default (None) means no reweighting. A sensible value for this parameter depends on the level of RFI in the data. Small values (eg. 2) result in aggressive reweighting and @@ -103,4 +96,11 @@ inputs: - (.)dist.yml - (.)out.yml -outputs: {} +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/gridding.yml b/cultcargo/genesis/pfb-imaging/latest/gridding.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/gridding.yml rename to cultcargo/genesis/pfb-imaging/latest/gridding.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/hci.yaml b/cultcargo/genesis/pfb-imaging/latest/hci.yaml new file mode 100644 index 0000000..58f1000 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/hci.yaml @@ -0,0 +1,155 @@ +inputs: + ms: + dtype: List[URI] + required: true + abbreviation: ms + info: Path to measurement set + scans: + dtype: List[int] + info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated + list 0,2 if running from CLI + ddids: + dtype: List[int] + info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated + list 0,2 if running from CLI + fields: + dtype: List[int] + info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list + 0,2 if running from CLI + freq-range: + dtype: str + info: Frequency range to image in Hz. Specify as a string with colon delimiter + eg. '1e9:1.1e9' + overwrite: + dtype: bool + default: false + info: Allow overwrite of output xds + transfer-model-from: + dtype: str + abbreviation: tmf + info: Name of dataset to use for model initialisation + data-column: + dtype: str + default: DATA + abbreviation: dc + info: Data column to image. Must be the same across MSs. Simple arithmetic is + supported eg. 'CORRECTED_DATA-MODEL_DATA'. When gains are present this column + will be corrected. + model-column: + dtype: str + abbreviation: mc + info: Model column to subtract from corrected data column. Useful to avoid high + time res degridding. + weight-column: + dtype: str + abbreviation: wc + info: Column containing natural weights. Must be the same across MSs + sigma-column: + dtype: str + abbreviation: sc + info: Column containing standard devations. Will be used to initialise natural + weights if detected. Must be the same across MSs + flag-column: + dtype: str + default: FLAG + abbreviation: fc + info: Column containing data flags. Must be the same across MSs + gain-table: + dtype: List[URI] + info: Path to Quartical gain table containing NET gains. There must be a table + for each MS and glob(ms) and glob(gt) should match up when running from CLI. + integrations-per-image: + dtype: int + abbreviation: ipi + default: 1 + info: Number of time integrations per image. Default (-1, 0, None) -> dataset + per scan. + channels-per-image: + dtype: int + abbreviation: cpi + default: -1 + info: Number of channels per image for gridding resolution. Default of (-1, 0, + None) implies a single dataset per spw. + precision: + dtype: str + default: double + choices: + - single + - double + info: Gridding precision + field-of-view: + dtype: float + abbreviation: fov + default: 1.0 + info: Field of view in degrees + super-resolution-factor: + dtype: float + default: 1.4 + abbreviation: srf + info: Will over-sample Nyquist by this factor at max frequency + cell-size: + dtype: float + info: Cell size in arc-seconds + nx: + dtype: int + info: Number of x pixels + ny: + dtype: int + info: Number of y pixels + robustness: + dtype: float + abbreviation: rob + info: Robustness factor for Briggs weighting. None means natural + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + l2-reweight-dof: + dtype: float + info: The degrees of freedom parameter for L2 reweighting. The default (None) + means no reweighting. A sensible value for this parameter depends on the level + of RFI in the data. Small values (eg. 2) result in aggressive reweighting and + should be avoided if the model is still incomplete. + progressbar: + dtype: bool + default: true + info: Display progress. Use --no-progressbar to deactivate. + output-format: + default: zarr + info: zarr or fits output + eta: + dtype: float + default: 1e-5 + info: Value to add to Hessian to make it invertible. Smaller values tend to fit + more noise and make the inversion less stable. + concat-time: + dtype: bool + default: false + info: Concatenate gains and times across scan + natural-grad: + dtype: bool + default: false + info: Compute naural gradient + memory_reporting: + dtype: bool + default: false + info: Report worker memory as tasks complete + check-ants: + dtype: bool + default: false + info: Check that ANTENNA1 and ANTENNA2 tables are consistent with the ANTENNA + table. + _include: + - (.)gridding.yml + - (.)out.yml + - (.)dist.yml + - (.)cgopts.yml + +outputs: + dir-out: + implicit: '{current.output-filename}_{current.product}.fds' + dtype: Directory + must_exist: false + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/init.yaml b/cultcargo/genesis/pfb-imaging/latest/init.yaml similarity index 57% rename from cultcargo/genesis/pfb-clean/latest/init.yaml rename to cultcargo/genesis/pfb-imaging/latest/init.yaml index 7d6177a..f5dc3f0 100644 --- a/cultcargo/genesis/pfb-clean/latest/init.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/init.yaml @@ -6,18 +6,18 @@ inputs: info: Path to measurement set scans: dtype: List[int] - info: List of SCAN_NUMBERS to image. Defaults to all. Input as string eg. '[0,2]' - if running from CLI + info: List of SCAN_NUMBERS to image. Defaults to all. Input as comma separated + list 0,2 if running from CLI ddids: dtype: List[int] - info: List of DATA_DESC_ID's to images. Defaults to all. Input as string eg. '[0,1]' - if running from CLI + info: List of DATA_DESC_ID's to images. Defaults to all. Input as comma separated + list 0,2 if running from CLI fields: dtype: List[int] - info: List of FIELD_ID's to image. Defaults to all. Input as string eg. '[0,1,2]' - if running from CLI + info: List of FIELD_ID's to image. Defaults to all. Input as comma separated list + 0,2 if running from CLI freq-range: - dtype: str + dtype: Optional[str] info: Frequency range to image in Hz. Specify as a string with colon delimiter eg. '1e9:1.1e9' overwrite: @@ -31,7 +31,9 @@ inputs: dtype: str default: DATA abbreviation: dc - info: Data column to image. Must be the same across MSs + info: Data column to image. Must be the same across MSs. Simple arithmetic is + supported eg. 'CORRECTED_DATA-MODEL_DATA'. When gains are present this column + will be corrected. weight-column: dtype: str abbreviation: wc @@ -47,13 +49,9 @@ inputs: abbreviation: fc info: Column containing data flags. Must be the same across MSs gain-table: - dtype: List[str] + dtype: List[URI] info: Path to Quartical gain table containing NET gains. There must be a table - for each MS and glob(ms) and glob(gt) should match up - gain-term: - dtype: str - default: NET - info: Which gain term to use. Default is NET + for each MS and glob(ms) and glob(gt) should match up when running from CLI. integrations-per-image: dtype: int abbreviation: ipi @@ -63,8 +61,9 @@ inputs: channels-per-image: dtype: int abbreviation: cpi - default: -1 - info: Number of channels per image. Default (-1, 0, None) -> dataset per spw. + default: 32 + info: Number of channels per image for degridding resolution. Any of (-1, 0, None) + implies single dataset per spw. precision: dtype: str default: double @@ -76,21 +75,46 @@ inputs: dtype: float default: 1.0 info: BDA decorrelation factor. Only has an effect if less than one - beam-model: - dtype: str - info: Beam model to use max-field-of-view: dtype: float - abbreviation: fov default: 3.0 - info: Field of view in degrees + abbreviation: max-fov + info: The maximum field of view that will be considered. Used to compute decorrelation + due to BDA. + beam-model: + dtype: str + info: Beam model to use. Only katbeam currently supported. chan-average: dtype: int default: 1 info: Average this number if channels together + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + progressbar: + dtype: bool + default: true + info: Display progress. Use --no-progressbar to deactivate. + memory-reporting: + dtype: bool + default: false + info: Report worker memory as tasks complete + check-ants: + dtype: bool + default: false + info: Check that ANTENNA1 and ANTENNA2 tables are consistent with the ANTENNA + table. _include: - - (.)dist.yml - (.)out.yml + - (.)dist.yml + +outputs: + xds-out: + implicit: '{current.output-filename}_{current.product}.xds' + dtype: Directory + must_exist: false -outputs: {} +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/clean.yaml b/cultcargo/genesis/pfb-imaging/latest/kclean.yaml similarity index 85% rename from cultcargo/genesis/pfb-clean/latest/clean.yaml rename to cultcargo/genesis/pfb-imaging/latest/kclean.yaml index c8c6f61..4d1f703 100644 --- a/cultcargo/genesis/pfb-clean/latest/clean.yaml +++ b/cultcargo/genesis/pfb-imaging/latest/kclean.yaml @@ -1,10 +1,5 @@ inputs: - nband: - dtype: int - required: true - abbreviation: nb - info: Number of imaging bands - postfix: + suffix: dtype: str default: main info: Can be used to specify a custom name for the image space data products. @@ -16,14 +11,6 @@ inputs: dtype: int default: 1 info: Perform dilation followed by erosion with structure element of this width - fits-mfs: - dtype: bool - default: true - info: Output MFS fits files - fits-cubes: - dtype: bool - default: false - info: Output fits cubes mop-flux: dtype: bool default: true @@ -33,19 +20,23 @@ inputs: dtype: float default: 0.65 info: Step size for flux mop. Should be between (0,1). A value of 1 is most aggressive. - nmiter: + niter: dtype: int default: 5 - abbreviation: nmiter + abbreviation: niter info: Number of major iterations + nthreads: + dtype: int + abbreviation: nt + info: Total number of threads to use. Defaults to half the total number available. threshold: dtype: float info: Absolute threshold at which to stop cleaning. By default it is set automatically using sigmathreshold parameter abbreviation: th - sigmathreshold: + rmsfactor: dtype: float - default: 2.0 + default: 3.0 info: Multiple of the rms at which to terminate the minor cycle gamma: dtype: float @@ -112,13 +103,16 @@ inputs: dtype: bool default: true info: Ensure residual decreases at every iteration - memory-greedy: - dtype: bool - default: false - info: Holds data in memory if set + _include: - (.)gridding.yml - - (.)dist.yml - (.)out.yml -outputs: {} +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml new file mode 100644 index 0000000..fd88c1d --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/model2comps.yaml @@ -0,0 +1,77 @@ +inputs: + overwrite: + dtype: bool + default: false + info: Allow overwrite of existing model + mds: + dtype: str + info: An optional input model to append to the model. Will be rendered to the + same resolution as the model in the dds. + nbasisf: + dtype: int + info: Order of interpolating polynomial for frequency axis. One less than the + number of bands by default. + nbasist: + dtype: int + default: 1 + info: Order of interpolating polynomial for time axis. This is hypothetical for + the time being. + nthreads: + dtype: int + abbreviation: nt + info: Number of threads + fit-mode: + dtype: str + default: Legendre + min-val: + dtype: float + info: Only fit components above this flux level + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products + model-name: + dtype: str + default: MODEL + abbreviation: mname + info: Name of model in mds + use-wsum: + dtype: bool + default: true + info: Use wsum as weights during fit + sigmasq: + dtype: float + default: 1e-10 + info: Multiple of the identity to add to the hessian for stability + model-out: + dtype: str + info: Optional explicit output name. Otherwise the default naming convention is + used. + out-format: + dtype: str + default: zarr + choices: + - zarr + - json + info: Format to dump model to. + out-freqs: + dtype: str + info: A string flow:fhigh:step of frequencies in hertz where the output cube needs + to be evaluated. + + _include: + - (.)out.yml + +outputs: + mds-out: + implicit: =IFSET(current.model-out, current.model-out, + '{current.output-filename}_{current.product}_{current.suffix}_{current.model-name}.mds') + dtype: Directory + must_exist: false + # fits-out: + # implicit: =IFSET(current.model-out, STRIPEXT(current.model-out)+.fits, '{current.output-filename}_{current.product}_{current.suffix}_{current.model-name}.mds') + # dtype: Directory + # must_exist: false + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/out.yml b/cultcargo/genesis/pfb-imaging/latest/out.yml similarity index 53% rename from cultcargo/genesis/pfb-clean/latest/out.yml rename to cultcargo/genesis/pfb-imaging/latest/out.yml index 09c02c4..aeb7b1c 100644 --- a/cultcargo/genesis/pfb-clean/latest/out.yml +++ b/cultcargo/genesis/pfb-imaging/latest/out.yml @@ -5,7 +5,6 @@ output-filename: info: Basename of output log-directory: dtype: str - required: true abbreviation: ldir info: Directory to write logs and performance reports to. product: @@ -14,3 +13,16 @@ product: default: I info: Imaging products to produce. Options are I, Q, U, V. Only single Stokes products are currently supported +fits-output-folder: + dtype: str + abbreviation: fitso + info: Optional path to write fits files to. Set to output-filename if not provided. + The same naming conventions apply. +fits-mfs: + dtype: bool + default: true + info: Output MFS fits files +fits-cubes: + dtype: bool + default: true + info: Output fits cubes diff --git a/cultcargo/genesis/pfb-clean/latest/pdopts.yml b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml similarity index 86% rename from cultcargo/genesis/pfb-clean/latest/pdopts.yml rename to cultcargo/genesis/pfb-imaging/latest/pdopts.yml index bdfad58..1006587 100644 --- a/cultcargo/genesis/pfb-clean/latest/pdopts.yml +++ b/cultcargo/genesis/pfb-imaging/latest/pdopts.yml @@ -1,11 +1,11 @@ pd-tol: - dtype: float - default: 1e-3 + dtype: List[float] + default: [3e-4] info: Tolreance of primal dual algorithm abbreviation: pdtol pd-maxit: dtype: int - default: 250 + default: 450 info: Maximum iterations for primal dual algorithm abbreviation: pdmaxit pd-verbose: @@ -15,6 +15,6 @@ pd-verbose: abbreviation: pdverb pd-report-freq: dtype: int - default: 10 + default: 50 info: Report frequency of primal dual algorithm abbreviation: pdrf diff --git a/cultcargo/genesis/pfb-clean/latest/pmopts.yml b/cultcargo/genesis/pfb-imaging/latest/pmopts.yml similarity index 100% rename from cultcargo/genesis/pfb-clean/latest/pmopts.yml rename to cultcargo/genesis/pfb-imaging/latest/pmopts.yml diff --git a/cultcargo/genesis/pfb-imaging/latest/restore.yaml b/cultcargo/genesis/pfb-imaging/latest/restore.yaml new file mode 100644 index 0000000..0a9f5ab --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/restore.yaml @@ -0,0 +1,50 @@ +inputs: + _include: + - (.)dist.yml + - (.)out.yml + model-name: + dtype: str + abbreviation: mname + default: MODEL + info: Name of model in dds + residual-name: + dtype: str + abbreviation: rname + default: RESIDUAL + info: Name of residual in dds + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products + outputs: + dtype: str + default: mMrRiI + info: Output products (m)odel, (r)esidual, (i)mage, (c)lean beam, (d)irty, (f)ft_residuals + (amplitude and phase will be produced). Use captitals to produce corresponding + cubes. + overwrite: + dtype: bool + default: true + info: Allow overwriting fits files + gausspar: + dtype: List[float] + info: Gaussian parameters (e-major, e-minor, position-angle) specifying the resolution + to restore images to. The major and minor axes need to be specified in units + of arcseconds and the position-angle in degrees. The default resolution is the + native resolution in each imaging band. This parameter can be used to homogenise + the resolution of the cubes. Set to (0,0,0) to use the resolution of the lowest + band. + inflate-factor: + dtype: float + default: 1.5 + info: Inflate the intrinsic resolution of the uniformly blurred image by this + amount. + drop-bands: + dtype: List[int] + # default: [] How to make the default an empty list? + info: List of bands to discard + +outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/sara.yaml b/cultcargo/genesis/pfb-imaging/latest/sara.yaml new file mode 100644 index 0000000..d277d9b --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/sara.yaml @@ -0,0 +1,116 @@ +inputs: + _include: + - (.)gridding.yml + - (.)pdopts.yml + - (.)pmopts.yml + - (.)cgopts.yml + - (.)out.yml + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products + bases: + dtype: str + info: 'Wavelet bases to use. Give as comma separated str eg. ' + default: self,db1,db2,db3 + abbreviation: bases + nlevels: + dtype: int + info: Wavelet decomposition level + default: 2 + abbreviation: nlevels + l1-reweight-from: + dtype: int + default: 5 + info: L1 reweighting will kick in either at convergence or after this many iterations. + Set to a negative value to disbale L1 reweighting. + hess-norm: + dtype: float + abbreviation: hess-norm + info: Spectral norm of hessian approximation + hess-approx: + dtype: str + default: psf + abbreviation: hess-approx + choices: [wgt, psf, direct] + info: Which Hessian approximation to use. Set to wgt for vis space approximation, + psf for zero-padded image space approximation and direct for direct inversion. + rmsfactor: + dtype: float + default: 1.0 + info: By default will threshold by rmsfactor*rms at every iteration + eta: + dtype: float + default: 1.0 + info: Will use eta*wsum to regularise the inversion of the Hessian approximation. + gamma: + dtype: float + default: 1.0 + info: Step size of update + alpha: + dtype: float + default: 2 + info: Controls how aggressively the l1reweighting is applied. Larger values correspond + to more agressive reweighting. + nbasisf: + dtype: int + info: Number of basis functions to use while fitting the frequency axis. Default + is to use the number of non-null imaging bands i.e. interpolation. + positivity: + dtype: int + default: 1 + # choices: + # - 0 + # - 1 + # - 2 + info: How to apply positivity constraint 0 -> no positivity, 1 -> normal positivity + constraint 2 -> strong positivity i.e. all pixels in a band > 0 + niter: + dtype: int + default: 10 + abbreviation: niter + info: Number of iterations. + nthreads: + dtype: int + abbreviation: nt + info: Total number of threads to use. Defaults to half the total number available. + tol: + dtype: float + default: 5e-4 + info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) + < tol + diverge-count: + dtype: int + default: 5 + info: Will terminate the algorithm if the rms increases this many times. Set to + larger than niter to disable this check. + skip-model: + dtype: bool + default: false + info: Skip model creation i.e. use with do-res to do-res only + rms-outside-model: + dtype: bool + default: false + info: Mask residual where model is non-zero when computing rms. This is not recommended + for largely non-empty fields. + init_factor: + dtype: float + default: 0.5 + info: Reduce the regularisation strength by this fraction at the outset. + verbosity: + dtype: int + default: 1 + info: Set to larger than 1 to report timings during residual computation + +outputs: + dds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}.dds' + dtype: Directory + must_exist: false + mds-out: + implicit: '{current.output-filename}_{current.product}_{current.suffix}_model.mds' + dtype: Directory + must_exist: false + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml new file mode 100644 index 0000000..955bdf0 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/smoovie.yaml @@ -0,0 +1,72 @@ +inputs: + fds: + dtype: str + info: Optional path to fds produced by fastim worker. Default naming convention + is used if not provided. + animate-axis: + dtype: str + default: time + choices: + - time + - freq + info: Which axis to animate + fps: + dtype: int + default: 8 + info: Frames per second + min-frac: + dtype: float + default: 3.5 + info: vmin will be set as min-frac * median(rms) + max-frac: + dtype: float + default: 3.5 + info: vmax will be set as min-frac * median(rms) + cmap: + dtype: str + default: gray + info: Any valid matplotlib cmap + text-colour: + dtype: str + default: cyan + info: Colour to render text on image + freq-range: + dtype: str + info: Frequency range to consider. Specify as a string with colon delimiter eg. + '1e9:1.1e9' + time-range: + dtype: str + info: Time range to consider in mjds Specify as a string with colon delimiter + eg. '5e7:5.5e7' + overwrite: + dtype: bool + default: false + info: Allow overwrite of output files with the same names. + out-format: + dtype: str + default: gif + choices: + - gif + - mp4 + info: The format to write movie out in. gifs are usually better quality but can + get quite large. mp4 quality not currently great with streamjoy. + optimize: + dtype: bool + default: false + info: To try and optimize the resulting gif. Only possible if gifsicle is installed. + crf: + dtype: int + default: 12 + info: Constant rate factor for controlling mp4 output quality. + scratch-dir: + dtype: URI + info: Streamjoy scratch directory. + + _include: + - (.)out.yml + - (.)dist.yml + +outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-imaging/latest/spotless.yaml b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml new file mode 100644 index 0000000..b5dd1d5 --- /dev/null +++ b/cultcargo/genesis/pfb-imaging/latest/spotless.yaml @@ -0,0 +1,143 @@ +inputs: + _include: + - (.)gridding.yml + - (.)pdopts.yml + - (.)pmopts.yml + - (.)cgopts.yml + - (.)dist.yml + - (.)out.yml + xds: + info: Optional explicit path to xds. Set using output-filename and product by + default. + mds: + info: Optional explicit path to mds. Set using output-filename, product and suffix + by default. + suffix: + dtype: str + default: main + info: Can be used to specify a custom name for the image space data products + robustness: + dtype: float + abbreviation: rob + default: 0.5 + info: Robustness factor for Briggs weighting. None means natural + bases: + dtype: str + info: 'Wavelet bases to use. Give as comma separated str eg. ' + default: self,db1,db2 + abbreviation: bases + nlevels: + dtype: int + info: Wavelet decomposition level + default: 3 + abbreviation: nlevels + l1-reweight-from: + dtype: int + default: 5 + info: L1 reweighting will kick in either at convergence or after this many iterations. + Set to a negative value to disbale L1 reweighting. + hess-norm: + dtype: float + abbreviation: hess-norm + info: Spectral norm of hessian approximation + hess-approx: + dtype: str + default: direct + abbreviation: hess-approx + choices: [wgt, psf, direct] + info: Which Hessian approximation to use. Set to wgt for vis space approximation, + psf for zero-padded image space approximation and direct for direct inversion. + rmsfactor: + dtype: float + default: 1.0 + info: By default will threshold by rmsfactor*rms at every iteration + gamma: + dtype: float + default: 1.0 + info: Step size of update + alpha: + dtype: float + default: 2 + info: Controls how aggressively the l1reweighting is applied. Larger values correspond + to more agressive reweighting. + positivity: + dtype: int + default: 1 + # choices: [0,1,2] + info: | + How to apply positivity constraint + 0 -> no positivity + 1 -> normal positivity constraint + 2 -> strong positivity i.e. all pixels in a band > 0 + niter: + dtype: int + default: 10 + abbreviation: niter + info: Number of iterations. + tol: + dtype: float + default: 5e-4 + info: Tolerance at which to terminate algorithm. Will stop when norm(x-xp)/norm(x) + < tol + diverge-count: + dtype: int + default: 5 + info: Will terminate the algorithm if the rms increases this many times. Set to + larger than niter to disable this check. + field-of-view: + dtype: float + abbreviation: fov + info: Field of view in degrees + metavar: DEG + super-resolution-factor: + dtype: float + default: 2 + abbreviation: srf + info: Will over-sample Nyquist by this factor at max frequency + cell-size: + dtype: float + info: Cell size in arc-seconds + nx: + dtype: int + info: Number of x pixels + ny: + dtype: int + info: Number of y pixels + psf-oversize: + dtype: float + default: 1.0 + abbreviation: psfo + info: Size of PSF relative to dirty image + target: + dtype: str + info: This can be predefined celestial objects known to astropy or a string in + the format 'HH:MM:SS,DD:MM:SS' (note the , delimiter) + l2-reweight-from: + dtype: int + default: 10 + info: Start doing L2 reweights from this iteration onwards. L2 reweights will + also be triggered if the algorithm converges, L1 reweighting has already kicked + in and l2-reweight-dof is set. + l2-reweight-dof: + dtype: float + default: 3.5 + info: The degrees of freedom parameter for performing L2 reweighting. L2 reweighting + is triggered if L1 reweighting is active and convergence is reached and will + be applied a maximum of max-l2-reweight times. + max-l2-reweight: + dtype: int + default: 5 + info: The maximum number of L2 reweighting steps. + overwrite: + dtype: bool + default: false + info: Hard reset the cache + eta: + dtype: float + default: 1.0 + info: Value to add to Hessian to make it invertible + +outputs: {} + +policies: + pass_missing_as_none: true diff --git a/cultcargo/genesis/pfb-clean/latest/uncabbedcabs.yml b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml similarity index 68% rename from cultcargo/genesis/pfb-clean/latest/uncabbedcabs.yml rename to cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml index e8aa5bb..f7e23d3 100644 --- a/cultcargo/genesis/pfb-clean/latest/uncabbedcabs.yml +++ b/cultcargo/genesis/pfb-imaging/latest/uncabbedcabs.yml @@ -25,14 +25,14 @@ pfb.degrid: _include: - (.)degrid.yaml -pfb.clean: - command: pfb.workers.clean.clean +pfb.kclean: + command: pfb.workers.kclean.kclean flavour: python policies: pass_missing_as_none: true _include: - - (.)clean.yaml + - (.)kclean.yaml pfb.restore: command: pfb.workers.restore.restore @@ -43,32 +43,32 @@ pfb.restore: _include: - (.)restore.yaml -pfb.fwdbwd: - command: pfb.workers.fwdbwd.fwdbwd +pfb.fluxtractor: + command: pfb.workers.fluxtractor.fluxtractor flavour: python policies: pass_missing_as_none: true _include: - - (.)fwdbwd.yaml + - (.)fluxtractor.yaml -pfb.forward: - command: pfb.workers.forward.forward +pfb.spotless: + command: pfb.workers.spotless.spotless flavour: python policies: pass_missing_as_none: true _include: - - (.)forward.yaml + - (.)spotless.yaml -pfb.spotless: - command: pfb.workers.spotless.spotless +pfb.sara: + command: pfb.workers.sara.sara flavour: python policies: pass_missing_as_none: true _include: - - (.)spotless.yaml + - (.)sara.yaml pfb.model2comps: command: pfb.workers.model2comps.model2comps @@ -78,3 +78,21 @@ pfb.model2comps: _include: - (.)model2comps.yaml + +pfb.hci: + command: pfb.workers.hci.hci + flavour: python + policies: + pass_missing_as_none: true + + _include: + - (.)hci.yaml + +pfb.smoovie: + command: pfb.workers.smoovie.smoovie + flavour: python + policies: + pass_missing_as_none: true + + _include: + - (.)smoovie.yaml diff --git a/cultcargo/genesis/pfb-clean/sync_config.py b/cultcargo/genesis/pfb-imaging/sync_config.py similarity index 72% rename from cultcargo/genesis/pfb-clean/sync_config.py rename to cultcargo/genesis/pfb-imaging/sync_config.py index c94d8d1..04f18f0 100644 --- a/cultcargo/genesis/pfb-clean/sync_config.py +++ b/cultcargo/genesis/pfb-imaging/sync_config.py @@ -5,13 +5,13 @@ yaml.indent(mapping=2, sequence=4, offset=2) import cultcargo -mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-clean/latest' +mpath = cultcargo.__file__.rstrip('__init__.py') + 'genesis/pfb-imaging/latest' -# import ipdb; ipdb.set_trace() -content_path = 'https://raw.githubusercontent.com/ratt-ru/pfb-clean/awskube/pfb/parser' +branch = 'main' +content_path = f'https://raw.githubusercontent.com/ratt-ru/pfb-imaging/{branch}/pfb/parser' # returns list of files in subfolder -response = requests.get('https://api.github.com/repos/ratt-ru/pfb-clean/contents/pfb/parser?ref=awskube') +response = requests.get(f'https://api.github.com/repos/ratt-ru/pfb-imaging/contents/pfb/parser?ref={branch}') for r in response.json(): name = r['name'] print(name) diff --git a/cultcargo/genesis/quartical/argument_schema.yaml b/cultcargo/genesis/quartical/argument_schema.yaml index d3a54a9..af3b369 100644 --- a/cultcargo/genesis/quartical/argument_schema.yaml +++ b/cultcargo/genesis/quartical/argument_schema.yaml @@ -423,3 +423,9 @@ dask: info: Which dask scheduler to use. The default, threads, is the most appropriate for non-cluster use. + + scheduler_plugin: + dtype: bool + default: true + info: + Enable or disable the dask scheduler plugin. diff --git a/cultcargo/genesis/quartical/gain_schema.yaml b/cultcargo/genesis/quartical/gain_schema.yaml index 0a462bb..266c23b 100644 --- a/cultcargo/genesis/quartical/gain_schema.yaml +++ b/cultcargo/genesis/quartical/gain_schema.yaml @@ -8,12 +8,15 @@ gain: - amplitude - delay - delay_and_offset + - delay_and_tec - phase - tec_and_offset - rotation_measure - rotation - crosshand_phase + - crosshand_phase_null_v - leakage + - parallactic_angle info: Type of gain to solve for. diff --git a/cultcargo/genesis/wsclean/wsclean-base.yml b/cultcargo/genesis/wsclean/wsclean-base.yml index cf0877b..b29fc9e 100644 --- a/cultcargo/genesis/wsclean/wsclean-base.yml +++ b/cultcargo/genesis/wsclean/wsclean-base.yml @@ -70,6 +70,11 @@ lib: dtype: List[int] policies: repeat: ',' + multiscale-max-scales: + dtype: int + multiscale-gain: + dtype: float + default: 0.15 multiscale-scale-bias: dtype: float scale: diff --git a/cultcargo/pfb-clean.yml b/cultcargo/pfb-clean.yml deleted file mode 100644 index 828a6c3..0000000 --- a/cultcargo/pfb-clean.yml +++ /dev/null @@ -1,49 +0,0 @@ -_include: - - genesis/cult-cargo-base.yml - - -vars: - cult-cargo: - pfb-clean: - image: - _use: vars.cult-cargo.images - name: pfb-clean - -cabs: - _include: genesis/pfb-clean/latest/uncabbedcabs.yml - - pfb-init: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-grid: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-degrid: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-clean: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-restore: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-fwdbwd: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-forward: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-spotless: - image: - _use: vars.cult-cargo.pfb-clean.image - - pfb-model2comps: - image: - _use: vars.cult-cargo.pfb-clean.image diff --git a/cultcargo/pfb-imaging.yml b/cultcargo/pfb-imaging.yml new file mode 100644 index 0000000..271bcd4 --- /dev/null +++ b/cultcargo/pfb-imaging.yml @@ -0,0 +1,57 @@ +_include: + - genesis/cult-cargo-base.yml + + +vars: + cult-cargo: + pfb-imaging: + image: + _use: vars.cult-cargo.images + name: pfb-imaging + +cabs: + _include: genesis/pfb-imaging/latest/uncabbedcabs.yml + + pfb.init: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.grid: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.degrid: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.kclean: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.restore: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.fluxtractor: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.spotless: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.sara: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.model2comps: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.hci: + image: + _use: vars.cult-cargo.pfb-imaging.image + + pfb.smoovie: + image: + _use: vars.cult-cargo.pfb-imaging.image diff --git a/pyproject.toml b/pyproject.toml index 3290733..a298899 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ packages = [{include = "cultcargo"}] [tool.poetry.dependencies] python = "^3.8" -stimela = "^2.0.1" +stimela = "^2.0.1.1" requests = "^2.0" [tool.poetry.scripts]