From 19ac224512637a4b6c289addabf86463ad6b84bd Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Sun, 6 Feb 2022 21:04:23 +0100 Subject: [PATCH 1/7] Allow speeding up of single shot processing; broader test coverage; more type annotations --- histgrinder/config.py | 12 +- histgrinder/engine.py | 27 +++- histgrinder/io/root.py | 8 +- histgrinder/make_sample_file.py | 15 +- histgrinder/transform.py | 57 ++++--- setup.py | 2 +- tests/test_badconfig.yaml | 6 + tests/test_badfunction.yaml | 6 + tests/test_badpattern.yaml | 6 + tests/test_badreturn.yaml | 6 + tests/test_badtype.yaml | 6 + tests/test_empty.yaml | 0 tests/test_functional.yaml | 49 +++++++ tests/test_functional_noprefix.yaml | 41 ++++++ tests/test_pieces.py | 41 ++++++ tests/test_run.py | 220 ++++++++++++++++++++++++++-- 16 files changed, 450 insertions(+), 52 deletions(-) create mode 100644 tests/test_badconfig.yaml create mode 100644 tests/test_badfunction.yaml create mode 100644 tests/test_badpattern.yaml create mode 100644 tests/test_badreturn.yaml create mode 100644 tests/test_badtype.yaml create mode 100644 tests/test_empty.yaml create mode 100644 tests/test_functional.yaml create mode 100644 tests/test_functional_noprefix.yaml create mode 100644 tests/test_pieces.py diff --git a/histgrinder/config.py b/histgrinder/config.py index 65fd737..9be15c3 100644 --- a/histgrinder/config.py +++ b/histgrinder/config.py @@ -1,10 +1,10 @@ # Configuration utilities -from typing import Union, IO, List, Any +from typing import Union, IO, List, Any, Mapping import logging class TransformationConfiguration(object): - def __init__(self, Input, Output, Function, Description, Parameters={}): + def __init__(self, Input: List[str], Output: List[str], Function: str, Description: str, Parameters: Mapping = {}): self.input = Input self.output = Output self.function = Function @@ -20,7 +20,7 @@ def __repr__(self): def read_configuration(f: Union[str, IO]) -> List[TransformationConfiguration]: import yaml if isinstance(f, str): - fobj = open(f) + fobj = open(f, 'r') else: fobj = f @@ -38,11 +38,11 @@ def read_configuration(f: Union[str, IO]) -> List[TransformationConfiguration]: def lookup_name(name: str) -> Any: import importlib - name = name.rsplit('.', 1) - return getattr(importlib.import_module(name[0]), name[1]) + spname = name.rsplit('.', 1) + return getattr(importlib.import_module(spname[0]), spname[1]) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover import sys for _ in read_configuration(sys.argv[1]): print(_) diff --git a/histgrinder/engine.py b/histgrinder/engine.py index 8708aa0..4db983c 100644 --- a/histgrinder/engine.py +++ b/histgrinder/engine.py @@ -26,6 +26,8 @@ def go(): choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO') + parser.add_argument('--defer', action='store_true', help='Defer processing of histograms until end of input loop') + parser.add_argument('--delaywrite', action='store_true', help='Write histograms at once at end of job') args = parser.parse_args() logging.basicConfig(level=args.loglevel, @@ -52,7 +54,7 @@ def go(): # Configure output om = lookup_name(args.outmodule)() - out_configuration = {'target': args.target} + out_configuration = {'target': args.target, 'delay': args.delaywrite} if args.prefix: out_configuration['prefix'] = args.prefix om.configure(out_configuration) @@ -61,18 +63,31 @@ def go(): log.info("Warmup") for obj in im.warmup(): for _ in transformers: - v = _.consider(obj) + _.consider(obj) - # Event loop log.info("Beginning loop") + eventloop(im, om, transformers, args, log) + log.info("Complete") + + +def eventloop(im, om, transformers, args, log): + # Event loop for obj in im: for _ in transformers: - v = _.consider(obj) + v = _.consider(obj, defer=args.defer) if v: om.publish(v) + if args.defer: + log.info("Processing deferred results") + for _ in transformers: + lv = _.transform() + for v in lv: + om.publish(v) - log.info("Complete") + if args.delaywrite: + log.info("Finalizing output") + om.finalize() -if __name__ == '__main__': +if __name__ == '__main__': # pragma: no cover go() diff --git a/histgrinder/io/root.py b/histgrinder/io/root.py index ee4b8f4..bd5474c 100644 --- a/histgrinder/io/root.py +++ b/histgrinder/io/root.py @@ -47,9 +47,7 @@ def iterate(self, dryrun) -> Generator[HistObject, None, None]: import os.path from collections import deque log = logging.getLogger(__name__) - infile = ROOT.TFile.Open(self.source) - if not infile: - raise ValueError(f"Unable to open input file {self.source}") + infile = ROOT.TFile.Open(self.source) # failure to open will raise OSError dirqueue = deque(['']) while dirqueue: dirname = dirqueue.popleft() @@ -111,7 +109,7 @@ def configure(self, options: Mapping[str, Any]) -> None: """ if 'target' not in options: raise ValueError("Must specify 'target' as an option " - "to ROOTInputModule") + "to ROOTOutputModule") self.target = options['target'] self.overwrite = bool(options.get('overwrite', True)) self.prefix = options.get('prefix', '/') @@ -158,7 +156,7 @@ def finalize(self) -> None: self._write() -if __name__ == '__main__': # pragma: no test +if __name__ == '__main__': # pragma: no cover import sys if len(sys.argv) != 3: print("Provide two arguments (input and output files)") diff --git a/histgrinder/make_sample_file.py b/histgrinder/make_sample_file.py index 3bf0cd2..4d1c1bb 100644 --- a/histgrinder/make_sample_file.py +++ b/histgrinder/make_sample_file.py @@ -1,6 +1,7 @@ -if __name__ == '__main__': # pragma: no test +if __name__ == '__main__': # pragma: no cover import ROOT import random + import array random.seed(42) f = ROOT.TFile.Open('example.root', 'RECREATE') @@ -11,4 +12,16 @@ for j in range(1000): h.Fill(random.gauss(0, 1)) h.Write() + + # make a graph + x = array.array('f', [1]) + y = array.array('f', [1]) + g = ROOT.TGraph(1, x, y) + g.Write('graph') + + # make a tree + t = ROOT.TTree('tree', 'tree') + t.Branch('x', x, 'x/F') + t.Fill() + t.Write() f.Close() diff --git a/histgrinder/transform.py b/histgrinder/transform.py index 9c65246..228136b 100644 --- a/histgrinder/transform.py +++ b/histgrinder/transform.py @@ -1,13 +1,14 @@ from .config import TransformationConfiguration, lookup_name from .HistObject import HistObject -from typing import Optional, List +from typing import DefaultDict, Optional, List, Tuple, Match, Dict +import re class Transformer(object): def __init__(self, tc: TransformationConfiguration): - import re import string self.tc = tc + self.matchqueue = set() # the number of histograms needed for a match self.inlength = len(tc.input) # regexes of input @@ -26,7 +27,7 @@ def __init__(self, tc: TransformationConfiguration): for output in self.tc.output]) # one dictionary for each input slot - self.hits = [{} for _ in range(len(self.inregexes))] + self.hits: List[Dict[Tuple[str], HistObject]] = [{} for _ in range(len(self.inregexes))] try: self.transform_function = lookup_name(tc.function) if not callable(self.transform_function): @@ -35,7 +36,7 @@ def __init__(self, tc: TransformationConfiguration): except Exception as e: raise ValueError(f"Unable to instantiate transformer because: {e}") - def consider(self, obj: HistObject, dryrun: bool = False) -> Optional[List[HistObject]]: + def consider(self, obj: HistObject, defer: bool = False) -> Optional[List[HistObject]]: """ Emit a new plot if we get a full match, otherwise None """ import logging log = logging.getLogger(__name__) @@ -48,27 +49,41 @@ def consider(self, obj: HistObject, dryrun: bool = False) -> Optional[List[HistO match = imatch if match is None: return None - # Return value list - rv = [] - - # Given a match, what first position histograms are relevant? - firstmatches = self._getMatchingFirstHists(match) - # Group the first position matches by integration variables - groupedfirstmatches = self._groupMatches(firstmatches) + self.matchqueue.add(match) + if defer: + return None + return self.transform() - # construct iterables & call functions - for tuplist in groupedfirstmatches.values(): - hci = HistCombinationIterable(self, tuplist) - if _fullyvalid(hci): - olist = self.transform_function(hci, **self.tc.parameters) - for i, ohist in enumerate(olist): - oname = self.tc.output[i].format(**dict(zip(self.regextupnames[0], tuplist[0]))) - rv.append(HistObject(oname, ohist)) + def transform(self) -> List[HistObject]: + # Return value list + rv = [] + firstmatchset = set() + for match in self.matchqueue: + # Given a match, what first position histograms are relevant? + firstmatches = self._getMatchingFirstHists(match) + firstmatchset.add(tuple(firstmatches)) + + for firstmatches in firstmatchset: + # Group the first position matches by integration variables + groupedfirstmatches = self._groupMatches(firstmatches) + + # construct iterables & call functions + for tuplist in groupedfirstmatches.values(): + hci = HistCombinationIterable(self, tuplist) + if _fullyvalid(hci): + olist = self.transform_function(hci, **self.tc.parameters) + if len(olist) != len(self.tc.output): + raise ValueError(f'Function {self.tc.function} gave {len(olist)} return values ' + f'but the YAML configuration specifies {len(self.tc.output)}.') + for i, ohist in enumerate(olist): + oname = self.tc.output[i].format(**dict(zip(self.regextupnames[0], tuplist[0]))) + rv.append(HistObject(oname, ohist)) + self.matchqueue.clear() return rv - def _getMatchingFirstHists(self, match): + def _getMatchingFirstHists(self, match: Match) -> List[Tuple[str]]: firstmatches = [] for tup in self.hits[0]: # does the tuple match in all spots where the pattern name matches, and @@ -80,7 +95,7 @@ def _getMatchingFirstHists(self, match): firstmatches.append(tup) return firstmatches - def _groupMatches(self, firstmatches): + def _groupMatches(self, firstmatches: List[Tuple[str]]) -> DefaultDict[Tuple[str], List[Tuple[str]]]: # group matches by self.outputnames import collections rv = collections.defaultdict(list) diff --git a/setup.py b/setup.py index e94bcac..fd35ad3 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,6 @@ ], python_requires='>=3.6', install_requires=['PyYAML>=5'], - tests_requires=['pytest','pytest-cov'], + tests_requires=['pytest', 'pytest-cov'], scripts=['bin/histgrinder'] ) diff --git a/tests/test_badconfig.yaml b/tests/test_badconfig.yaml new file mode 100644 index 0000000..1665994 --- /dev/null +++ b/tests/test_badconfig.yaml @@ -0,0 +1,6 @@ +--- +# Configuration has typo +Inputq: [ 'gaussians/gaus_(?P[23])(?P\d)', 'gaussians/gaus_5(?P\d)' ] +Output: [ 'gauDiv_{id0}{id}' ] +Function: math.pi +Description: Testing1 \ No newline at end of file diff --git a/tests/test_badfunction.yaml b/tests/test_badfunction.yaml new file mode 100644 index 0000000..6ac14a5 --- /dev/null +++ b/tests/test_badfunction.yaml @@ -0,0 +1,6 @@ +--- +# Function is not a function +Input: [ 'gaussians/gaus_(?P[23])(?P\d)', 'gaussians/gaus_5(?P\d)' ] +Output: [ 'gauDiv_{id0}{id}' ] +Function: math.pi +Description: Testing1 \ No newline at end of file diff --git a/tests/test_badpattern.yaml b/tests/test_badpattern.yaml new file mode 100644 index 0000000..4bb6e90 --- /dev/null +++ b/tests/test_badpattern.yaml @@ -0,0 +1,6 @@ +--- +# Bad order +Input: [ 'gaussians/gaus_5(?P\d)', 'gaussians/gaus_(?P[23])(?P\d)' ] +Output: [ 'gauDiv_{id0}{id}' ] +Function: histgrinder.example.transform_function_divide_ROOT +Description: Testing1 \ No newline at end of file diff --git a/tests/test_badreturn.yaml b/tests/test_badreturn.yaml new file mode 100644 index 0000000..9d12573 --- /dev/null +++ b/tests/test_badreturn.yaml @@ -0,0 +1,6 @@ +--- +# Function has wrong number of return arguments +Input: [ 'gaussians/gaus_(?P[23])(?P\d)', 'gaussians/gaus_5(?P\d)' ] +Output: [ 'gauDiv_{id0}{id}' ] +Function: histgrinder.example.nop +Description: Testing1 \ No newline at end of file diff --git a/tests/test_badtype.yaml b/tests/test_badtype.yaml new file mode 100644 index 0000000..b78b364 --- /dev/null +++ b/tests/test_badtype.yaml @@ -0,0 +1,6 @@ +--- +# Function returns a type not handled by histgrinder +Input: [ 'gaussians/gaus_(?P[23])(?P\d)', 'gaussians/gaus_5(?P\d)' ] +Output: [ 'gauDiv_{id0}{id}' ] +Function: tests.test_run.returncppstr +Description: Testing1 \ No newline at end of file diff --git a/tests/test_empty.yaml b/tests/test_empty.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_functional.yaml b/tests/test_functional.yaml new file mode 100644 index 0000000..bfc2067 --- /dev/null +++ b/tests/test_functional.yaml @@ -0,0 +1,49 @@ +# Separate blocks with --- +--- +# This function takes the ratio of some histograms +Input: [ 'gaussians/gaus_(?P[23])(?P\d)', 'gaussians/gaus_5(?P\d)' ] +Output: [ 'gauDiv_{id0}{id}' ] +Function: histgrinder.example.transform_function_divide_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing1 +--- +# This function sums the ratio of some histograms +Input: [ 'gaussians/gaus_(?P\d)(?P\d)', 'gaussians/gaus_(?P\d)(?P\d)' ] +Output: [ 'gauDiv2_{id0}' ] +Function: histgrinder.example.transform_function_divide2_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing2 +--- +# This function combines multiple histograms into one +Input: [ 'gaussians/gaus_(?P\d+)' ] +Output: [ 'gauRMS' ] +Function: histgrinder.example.transform_function_rms_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing3 +--- +# This function takes the ratio of one pair of histograms +Input: [ 'gaussians/gaus_72', 'gaussians/gaus_18' ] +Output: [ 'gauDivSpecial' ] +Function: histgrinder.example.transform_function_divide_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing4 +--- +# This function takes the ratio of one pair of histograms, but reversed from above +Input: [ 'gaussians/gaus_18', 'gaussians/gaus_72' ] +Output: [ 'gauDivSpecialInv' ] +Function: histgrinder.example.transform_function_divide_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing5 +--- +# This function does nothing +Input: [ 'gaussians/graph' ] +Output: [ ] +Function: histgrinder.example.nop +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing5 diff --git a/tests/test_functional_noprefix.yaml b/tests/test_functional_noprefix.yaml new file mode 100644 index 0000000..becc522 --- /dev/null +++ b/tests/test_functional_noprefix.yaml @@ -0,0 +1,41 @@ +# Separate blocks with --- +--- +# This function takes the ratio of some histograms +Input: [ 'prefix/gaussians/gaus_(?P[23])(?P\d)', 'prefix/gaussians/gaus_5(?P\d)' ] +Output: [ 'prefix/gauDiv_{id0}{id}' ] +Function: histgrinder.example.transform_function_divide_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing1 +--- +# This function sums the ratio of some histograms +Input: [ 'prefix/gaussians/gaus_(?P\d)(?P\d)', 'prefix/gaussians/gaus_(?P\d)(?P\d)' ] +Output: [ 'prefix/gauDiv2_{id0}' ] +Function: histgrinder.example.transform_function_divide2_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing2 +--- +# This function combines multiple histograms into one +Input: [ 'prefix/gaussians/gaus_(?P\d+)' ] +Output: [ 'prefix/gauRMS' ] +Function: histgrinder.example.transform_function_rms_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing3 +--- +# This function takes the ratio of one pair of histograms +Input: [ 'prefix/gaussians/gaus_72', 'prefix/gaussians/gaus_18' ] +Output: [ 'prefix/gauDivSpecial' ] +Function: histgrinder.example.transform_function_divide_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing4 +--- +# This function takes the ratio of one pair of histograms, but reversed from above +Input: [ 'prefix/gaussians/gaus_18', 'prefix/gaussians/gaus_72' ] +Output: [ 'prefix/gauDivSpecialInv' ] +Function: histgrinder.example.transform_function_divide_ROOT +#Here is how you would pass parameters +#Parameters: { variable: eta, binedges: [-2.5,-2.0,-1.5,-1.0,-0.5,0.0,0.5,1.0,1.5,2.0,2.5] } +Description: Testing5 diff --git a/tests/test_pieces.py b/tests/test_pieces.py new file mode 100644 index 0000000..c3cf36c --- /dev/null +++ b/tests/test_pieces.py @@ -0,0 +1,41 @@ +import pytest + + +def test_rootio(): + pytest.importorskip("ROOT") + + from histgrinder.io.root import ROOTInputModule + rim = ROOTInputModule() + with pytest.raises(ValueError): + rim.configure({}) + + from histgrinder.io.root import ROOTOutputModule + rom = ROOTOutputModule() + with pytest.raises(ValueError): + rom.configure({}) + + # this is just for coverage completeness (tests the branch if the ROM.queue is empty on write) + rom.configure({'target': ''}) + rom.finalize() + + +def test_badconfig(): + from histgrinder.config import read_configuration + c = read_configuration('tests/test_badconfig.yaml') + assert c == [] + + +def test_read_config_from_filelike_object(): + from histgrinder.config import read_configuration + c = read_configuration(open('tests/test_functional.yaml')) + print(repr(c[0])) + assert (repr(c[0]) == r"""Description: Testing1 +Input: ['gaussians/gaus_(?P[23])(?P\\d)', 'gaussians/gaus_5(?P\\d)'] +Output: ['gauDiv_{id0}{id}'] +Function: histgrinder.example.transform_function_divide_ROOT, Parameters: {}""") + + +def test_read_nonexistent_config(): + from histgrinder.config import read_configuration + with pytest.raises(FileNotFoundError): + read_configuration('tests/missing.yaml') diff --git a/tests/test_run.py b/tests/test_run.py index 4327bf3..b9a3f93 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,28 +1,224 @@ +from subprocess import CalledProcessError import pytest -def test_run(): - ROOT = pytest.importorskip("ROOT") +def content_verify(newfile=False): + # ROOT should already have been checked to exist + import ROOT + # verify contents + f = ROOT.TFile.Open("new.root" if newfile else "example.root") + d = f.Get('prefix') + # did we create new histograms? + assert len(d.GetListOfKeys()) == (32 if newfile else 33) + # correct size? + assert f.Get("prefix/gauRMS").GetEntries() == 100 + return True + + +def returncppstr(inputs): + import cppyy + return [cppyy.gbl.std.string('abc')] + + +def test_run_stream(): + pytest.importorskip("ROOT") import subprocess chk = subprocess.run("python -m histgrinder.make_sample_file", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) chk.check_returncode() - chk = subprocess.run("wget https://raw.githubusercontent.com/ponyisi/histogram_postprocessing/master/resources/example.yaml -O example.yaml", # noqa: E501 + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_functional.yaml --prefix prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + return content_verify() + + +def test_run_newfile(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) chk.check_returncode() - chk = subprocess.run("python -m histgrinder.engine example.root example.root -c example.yaml --prefix prefix", + chk = subprocess.run("python -m histgrinder.engine example.root new.root -c tests/test_functional.yaml --prefix prefix", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) chk.check_returncode() - # verify contents - f = ROOT.TFile.Open("example.root") - d = f.Get("prefix") - # did we create new histograms? - assert len(d.GetListOfKeys()) == 33 - # correct size? - assert f.Get("prefix/gauRMS").GetEntries() == 100 - return True + return content_verify(newfile=True) + + +def test_run_noprefix(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root -c tests/test_functional_noprefix.yaml", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + return content_verify() + + +def test_run_absprefix(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_functional.yaml --prefix /prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + return content_verify() + + +def test_run_defer(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_functional.yaml --prefix prefix --defer", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + return content_verify() + + +def test_run_delaywrite(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_functional.yaml --prefix prefix --delaywrite", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + return content_verify() + + +def test_run_badpattern(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_badpattern.yaml --prefix prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + assert b'ValueError: Pattern specification problem: all named groups must be given in the first pattern' in chk.stdout + with pytest.raises(subprocess.CalledProcessError): + chk.check_returncode() + + +def test_run_badfunction(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_badfunction.yaml --prefix prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + assert b'ValueError: math.pi does not appear to be callable' in chk.stdout + with pytest.raises(subprocess.CalledProcessError): + chk.check_returncode() + + +def test_run_noinput(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.engine missing.root missing.root " + "-c tests/test_functional.yaml --prefix prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + assert b'Failed to open file missing.root' in chk.stdout + with pytest.raises(subprocess.CalledProcessError): + chk.check_returncode() + + +def test_run_bad_prefix(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.engine example.root example.root " + "-c tests/test_functional.yaml --prefix doesNotExist", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + assert b'Access to invalid directory' in chk.stdout + chk.check_returncode() + + +def test_run_nocode(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root -c tests/test_empty.yaml --prefix /prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + +def test_run_badtype(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root -c tests/test_badtype.yaml --prefix prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + assert b'ERROR | ROOT output: unsupported object type string' in chk.stdout + + +def test_run_badreturn(): + pytest.importorskip("ROOT") + + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + chk = subprocess.run("python -m histgrinder.engine example.root example.root -c tests/test_badreturn.yaml --prefix prefix", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + assert b'Function histgrinder.example.nop gave 0 return values but the YAML configuration specifies 1' in chk.stdout + with pytest.raises(CalledProcessError): + chk.check_returncode() From a9a2e8bfdcad5564d96e7aa51412ce3e62ebaf55 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Sun, 6 Feb 2022 21:09:55 +0100 Subject: [PATCH 2/7] Test against more recent Python versions --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3288686..5902339 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2 From 9bfecb139d2d5d3b072bcab86b2a7d6526463f94 Mon Sep 17 00:00:00 2001 From: ponyisi Date: Sun, 6 Feb 2022 18:26:07 -0500 Subject: [PATCH 3/7] Edit python versions --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5902339..5787587 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 From 779b08089e516401c3caf7b9fc876fbc19a504b9 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Mon, 7 Feb 2022 00:53:01 +0100 Subject: [PATCH 4/7] Add conda-forge channel at start --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5787587..ba46e76 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -41,6 +41,7 @@ jobs: uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true + channels: conda-forge python-version: ${{ matrix.python-version }} - name: Install conda dependencies shell: bash -l {0} From 0bea2ec2b1d56a466bd986e78908b83ddef6df5f Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Mon, 7 Feb 2022 00:59:20 +0100 Subject: [PATCH 5/7] Further update --- .github/workflows/python-package.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ba46e76..07647d4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2 @@ -46,8 +46,12 @@ jobs: - name: Install conda dependencies shell: bash -l {0} run: | - conda install -c conda-forge root conda install pytest pyyaml pytest-cov + conda install -c conda-forge root + - name: Build + shell: bash -l {0} + run: | + pip install -e . - name: Test with pytest shell: bash -l {0} run: | From 47ea97e221d56f10e68ef69117f188c4c6fce425 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Mon, 7 Feb 2022 02:00:42 +0100 Subject: [PATCH 6/7] 100% coverage --- tests/test_empty.yaml | 0 tests/test_pieces.py | 11 +++++++++++ tests/test_run.py | 24 ++++++++---------------- 3 files changed, 19 insertions(+), 16 deletions(-) delete mode 100644 tests/test_empty.yaml diff --git a/tests/test_empty.yaml b/tests/test_empty.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_pieces.py b/tests/test_pieces.py index c3cf36c..4c20654 100644 --- a/tests/test_pieces.py +++ b/tests/test_pieces.py @@ -18,6 +18,17 @@ def test_rootio(): rom.configure({'target': ''}) rom.finalize() + # this is also for coverage completeness (see that we get everything if no selector is set for the input module) + import subprocess + chk = subprocess.run("python -m histgrinder.make_sample_file", + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(chk.stdout) + chk.check_returncode() + + rim.configure({'source': 'example.root'}) + inputs = list(rim.iterate(dryrun=True)) + assert len(inputs) == 101 + def test_badconfig(): from histgrinder.config import read_configuration diff --git a/tests/test_run.py b/tests/test_run.py index b9a3f93..12617e7 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -41,10 +41,13 @@ def test_run_newfile(): pytest.importorskip("ROOT") import subprocess + import os.path chk = subprocess.run("python -m histgrinder.make_sample_file", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) chk.check_returncode() + if os.path.exists('new.root'): + os.remove('new.root') chk = subprocess.run("python -m histgrinder.engine example.root new.root -c tests/test_functional.yaml --prefix prefix", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) @@ -73,17 +76,20 @@ def test_run_absprefix(): pytest.importorskip("ROOT") import subprocess + import os.path chk = subprocess.run("python -m histgrinder.make_sample_file", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) chk.check_returncode() - chk = subprocess.run("python -m histgrinder.engine example.root example.root " + if os.path.exists('new.root'): + os.remove('new.root') + chk = subprocess.run("python -m histgrinder.engine example.root new.root " "-c tests/test_functional.yaml --prefix /prefix", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(chk.stdout) chk.check_returncode() - return content_verify() + return content_verify(newfile=True) def test_run_defer(): @@ -179,20 +185,6 @@ def test_run_bad_prefix(): chk.check_returncode() -def test_run_nocode(): - pytest.importorskip("ROOT") - - import subprocess - chk = subprocess.run("python -m histgrinder.make_sample_file", - shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(chk.stdout) - chk.check_returncode() - chk = subprocess.run("python -m histgrinder.engine example.root example.root -c tests/test_empty.yaml --prefix /prefix", - shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(chk.stdout) - chk.check_returncode() - - def test_run_badtype(): pytest.importorskip("ROOT") From 68b8f6edccbd9682c36d4cc2fd5d05f0987bc83f Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Mon, 7 Feb 2022 03:12:00 +0100 Subject: [PATCH 7/7] Don't run py 3.6 tests (conda takes forever to resolve) --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 07647d4..28bb4d7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, "3.10"] + python-version: [3.7, 3.8, 3.9, "3.10"] steps: - uses: actions/checkout@v2