Skip to content

Commit

Permalink
change config parameter name and default value using Mike's review su…
Browse files Browse the repository at this point in the history
…ggestions
  • Loading branch information
jchiang87 committed Feb 8, 2024
1 parent 0a6e956 commit eb0f63b
Show file tree
Hide file tree
Showing 3 changed files with 9 additions and 9 deletions.
4 changes: 2 additions & 2 deletions config/imsim-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -126,14 +126,14 @@ image:

# The objects are processed in batches.
# If checkpointing is turned on, then the checkpoint file will be updated after every
# group of checkpoint_sampling_factor batches. This factor helps lower the overall checkpoint
# group of nbatch_per_checkpoint batches. This factor helps lower the overall checkpoint
# rate when running thousands of processes concurrently; otherwise, the contention
# from lots of concurrent checkpoint writes can overwhelm the file system.
# Even if not checkpointing, using batches helps keep the memory down, since a bunch of
# temporary data can be purged after each batch.
# The default number of batches is 100, but you can change it if desired.
nbatch: 100
checkpoint_sampling_factor: 10
nbatch_per_checkpoint: 1

det_name: "$det_name"

Expand Down
6 changes: 3 additions & 3 deletions imsim/lsst_image.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger):
opt = { 'size': int , 'xsize': int , 'ysize': int, 'dtype': None,
'apply_sky_gradient': bool, 'apply_fringing': bool,
'boresight': galsim.CelestialCoord, 'camera': str, 'nbatch': int,
'checkpoint_sampling_factor': int}
'nbatch_per_checkpoint': int}
params = GetAllParams(config, base, req=req, opt=opt, ignore=ignore+extra_ignore)[0]

# Let the user override the image size
Expand Down Expand Up @@ -87,7 +87,7 @@ def setup(self, config, base, image_num, obj_num, ignore, logger):
try:
self.checkpoint = galsim.config.GetInputObj('checkpoint', config, base, 'LSST_Image')
self.nbatch = params.get('nbatch', 100)
self.checkpoint_sampling_factor = params.get('checkpoint_sampling_factor', 10)
self.nbatch_per_checkpoint = params.get('nbatch_per_checkpoint', 1)
except galsim.config.GalSimConfigError:
self.checkpoint = None
# Batching is also useful for memory reasons, to limit the number of stamps held
Expand Down Expand Up @@ -231,7 +231,7 @@ def buildImage(self, config, base, image_num, obj_num, logger):
base.get('extra_builder',None))
logger.warning('File %d: Completed batch %d with objects [%d, %d)',
base.get('file_num', 0), batch+1, start_obj_num, end_obj_num)
if (batch % self.checkpoint_sampling_factor == 0
if (batch % self.nbatch_per_checkpoint == 0
or batch + 1 == nbatch):
self.checkpoint.save(chk_name, data)
logger.warning('Wrote checkpoint data to %s', self.checkpoint.file_name)
Expand Down
8 changes: 4 additions & 4 deletions tests/test_checkpoint.py
Original file line number Diff line number Diff line change
Expand Up @@ -214,9 +214,9 @@ def test_checkpoint_image():
np.testing.assert_array_equal(centroid5, centroid2)


def test_checkpoint_output_sampling():
def test_nbatch_per_checkpoint():
"""Test that the final checkpoint files written with two different values
of checkpoint_sampling_factor, both not factors of nbatch, produce the same
of nbatch_per_checkpoint, both not factors of nbatch, produce the same
checkpoint output."""
wcs = galsim.PixelScale(0.2)
config = {
Expand Down Expand Up @@ -251,7 +251,7 @@ def test_checkpoint_output_sampling():
if os.path.exists(checkpoint_0):
os.remove(checkpoint_0)
config['image_num'] = 0
config['image']['checkpoint_sampling_factor'] = 11
config['image']['nbatch_per_checkpoint'] = 11
galsim.config.ProcessInput(config)
image0 = galsim.config.BuildImage(config)
with open(checkpoint_0, 'rb') as fobj:
Expand All @@ -261,7 +261,7 @@ def test_checkpoint_output_sampling():
if os.path.exists(checkpoint_1):
os.remove(checkpoint_1)
config['image_num'] = 1
config['image']['checkpoint_sampling_factor'] = 13
config['image']['nbatch_per_checkpoint'] = 13
galsim.config.ProcessInput(config)
image1 = galsim.config.BuildImage(config)
with open(checkpoint_1, 'rb') as fobj:
Expand Down

0 comments on commit eb0f63b

Please sign in to comment.