diff --git a/.ci/format_script.sh b/.ci/format_script.sh index 6bb51fa51291c..67503dd8710de 100755 --- a/.ci/format_script.sh +++ b/.ci/format_script.sh @@ -2,7 +2,6 @@ set -ex -BASE_COMMIT=$(git rev-parse $TRAVIS_BRANCH) echo "Running clang-format against branch $TRAVIS_BRANCH, with hash $BASE_COMMIT" COMMIT_FILES=$(git diff --name-only $BASE_COMMIT | grep -i -v LinkDef) RESULT_OUTPUT="$(git-clang-format --commit $BASE_COMMIT --diff --binary `which clang-format` $COMMIT_FILES)" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c1de82aa0e68f..657185aebf38f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -37,6 +37,7 @@ /tree/ @pcanal /tutorials/ @couet /tree/dataframe @eguiraud +/tree/ntuple/v7/ @jblomer # Projects that span over several code modules: /bindings/r/ @omazapa diff --git a/.github/workflows/code_analysis.yml b/.github/workflows/code_analysis.yml new file mode 100644 index 0000000000000..4b36e6597d9c4 --- /dev/null +++ b/.github/workflows/code_analysis.yml @@ -0,0 +1,23 @@ +name: clang-tools code analysis + +on: pull_request + # push: + # branches: [ $default-branch ] + # pull_request: + # branches: [ $default-branch ] + +jobs: + clang-format: + runs-on: ubuntu-latest + env: + TRAVIS_BRANCH: ${{ github.base_ref }} + TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }} + BASE_COMMIT: ${{ github.event.pull_request.base.sha }} + steps: + - uses: actions/checkout@v2 + - name: Fetch base_ref HEAD + run: git fetch --depth=1 origin +refs/heads/${{github.base_ref}}:refs/remotes/origin/${{github.base_ref}} + - name: install clang-format + run: sudo apt-get install -y clang-format + - name: run clang-format script + run: .ci/format_script.sh diff --git a/.travis.yml b/.travis.yml index 8041119e0bef9..180f5304daec9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,7 +23,11 @@ matrix: include: - env: TOOL=clang-format script: &format_script - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then .ci/format_script.sh; fi + - | + if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then + export BASE_COMMIT=$(git rev-parse $TRAVIS_BRANCH) + .ci/format_script.sh + fi - env: TOOL=clang-tidy-analyzer before_script: ©_headers diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a04e93c36e2d..d628452d0479f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -118,7 +118,7 @@ endif() #---Try to download a file to check internet connection----------------------------------------- message(STATUS "Checking internet connectivity...") -file(DOWNLOAD https://root.cern/files/test.txt ${CMAKE_CURRENT_BINARY_DIR}/test.txt +file(DOWNLOAD https://root.cern/files/cmake_connectivity_test.txt ${CMAKE_CURRENT_BINARY_DIR}/cmake_connectivity_test.txt TIMEOUT 5 STATUS DOWNLOAD_STATUS ) # Get the status code from the download status @@ -128,7 +128,7 @@ if(${STATUS_CODE} EQUAL 0) # Succcess message(STATUS "Yes") # Now let's delete the file - file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/test.txt) + file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/cmake_connectivity_test.txt) set(NO_CONNECTION FALSE) else() # Error @@ -619,15 +619,22 @@ if(testing) find_package(Git REQUIRED) if(IS_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/.git) - execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD + execute_process(COMMAND ${GIT_EXECUTABLE} for-each-ref --points-at=HEAD --count=1 --format=%\(refname:short\) WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE GIT_BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE) else() set(GIT_BRANCH v${ROOT_MAJOR_VERSION}-${ROOT_MINOR_VERSION}-${ROOT_PATCH_VERSION}) endif() + # Resolve the `latest-stable` branch to the latest merged head/tag + if("${GIT_BRANCH}" STREQUAL "latest-stable") + execute_process(COMMAND ${GIT_EXECUTABLE} for-each-ref --points-at=latest-stable^2 --format=%\(refname:short\) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + OUTPUT_VARIABLE GIT_BRANCH OUTPUT_STRIP_TRAILING_WHITESPACE) + endif() + execute_process(COMMAND - ${GIT_EXECUTABLE} ls-remote --heads + ${GIT_EXECUTABLE} ls-remote --heads --tags https://github.com/root-project/roottest.git ${GIT_BRANCH} OUTPUT_VARIABLE ROOTTEST_GIT_BRANCH) diff --git a/README/CREDITS b/README/CREDITS index 414945010fa98..4c3a543d86e42 100644 --- a/README/CREDITS +++ b/README/CREDITS @@ -615,8 +615,9 @@ E: loizides@MIT.EDU D: PROOF testing and debugging N: Javier Lopez-Gomez -E: jalopezg@inf.uc3m.es -D: Cling support for redefinitions +E: j.lopez@cern.ch +D: Cling support for redefinitions; cling improvements +D: RNTuple/DAOS integration N: Johan Lundberg E: johan.lundberg@cern.ch diff --git a/README/ReleaseNotes/v626/index.md b/README/ReleaseNotes/v626/index.md index e587ddf6cf9e2..4459b199595bd 100644 --- a/README/ReleaseNotes/v626/index.md +++ b/README/ReleaseNotes/v626/index.md @@ -23,10 +23,13 @@ The following people have contributed to this new version: Andrei Gheata, CERN/SFT,\ Enrico Guiraud, CERN/SFT,\ Jonas Hahnfeld, CERN/SFT,\ + Ivan Kabadzhov, CERN/SFT,\ Sergey Linev, GSI,\ + Javier Lopez-Gomez, CERN/SFT,\ Pere Mato, CERN/SFT,\ Lorenzo Moneta, CERN/SFT,\ Axel Naumann, CERN/SFT,\ + Vincenzo Eduardo Padulano, CERN/SFT and UPV,\ Max Orok, U Ottawa,\ Alexander Penev, University of Plovdiv,\ Danilo Piparo, CERN/SFT,\ @@ -49,13 +52,44 @@ The following people have contributed to this new version: ## Core Libraries +### Interpreter + +- As of v6.26, cling diagnostic messages can be redirected to the ROOT error handler. Users may enable/disable this via `TCling::ReportDiagnosticsToErrorHandler()`, e.g. +```cpp +root [1] gInterpreter->ReportDiagnosticsToErrorHandler(); +root [2] int f() { return; } +Error in : ROOT_prompt_2:1:11: non-void function 'f' should return a value [-Wreturn-type] +int f() { return; } + ^ +``` +More details at [PR #8737](https://github.com/root-project/root/pull/8737). ## I/O Libraries +- `TDirectory::WriteObject` now always saves the object's title to the file if it is derived from `TObject` (PR [#8394](https://github.com/root-project/root/pull/8934)). + +### Command line utilities + +- `rootls` now follows the same logic of `TFile::ls()` to print the key cycle number and its tag when listing contents of a file with the `-l` option (PR [#7878](https://github.com/root-project/root/pull/7878)): +``` +$: rootls -l https://root.cern/files/ttree_read_imt.root +TTree Mar 13 17:17 2019 TreeIMT;2 "TTree for IMT test" [current cycle] +TTree Mar 13 17:17 2019 TreeIMT;1 "TTree for IMT test" [backup cycle] +``` +- `root` will now error on receiving unrecognized options, similarly to other command line tools (PR [#8868](https://github.com/root-project/root/pull/8868)): +``` +$: root --random -z --nonexistingoption +root: unrecognized option '--random' +root: unrecognized option '-z' +root: unrecognized option '--nonexistingoption' +Try 'root --help' for more information. +``` ## TTree Libraries - `TTreeReader::GetEntryStatus` now always reports `kEntryBeyondEnd` after an event loop correctly completes. In previous versions, it could sometime return `kEntryNotFound` even for well-behaved event loops. +- Add `TEntryList::AddSubList` to specifically add a sub-list to the main list of entries. Consequently, add also a new option `"sync"` in `TChain::SetEntryList` to connect the sub-trees of the chain to the sub-lists of the entry list in lockstep (PR [#8660](https://github.com/root-project/root/pull/8660)). +- Add `TEntryList::EnterRange` to add all entries in a certain range `[start, end)` to the entry list (PR [#8740](https://github.com/root-project/root/pull/8740)). ## RDataFrame @@ -64,8 +98,42 @@ The following people have contributed to this new version: - Add `Redefine` to the `RDataFrame` interface, which allows to overwrite the value of an existing column. - Add `Describe` to the `RDataFrame` interface, which allows to get useful information, e.g. the columns and their types. - Add `DescribeDataset` to the `RDataFrame` interface, which allows to get information about the dataset (subset of the output of Describe()). -- `Book` now suports just-in-time compilation, i.e. it can be called without passing the column types as template parameters (with some performance penalty, as usual). -- As an aid to `RDataSource` implementations with which collection sizes can be retrieved more efficiently than the full collection, `#var` can now be used as a short-hand notation for column name `R_rdf_sizeof_var` +- Add [DefinePerSample](https://root.cern/doc/master/classROOT_1_1RDF_1_1RInterface.html#a29d77593e95c0f84e359a802e6836a0e), a method which makes it possible to define columns based on the sample and entry range being processed. It is also a useful way to register callbacks that should only be called when the input dataset/TTree changes. +- `Book` now supports just-in-time compilation, i.e. it can be called without passing the column types as template parameters (with some performance penalty, as usual). +- As an aid to `RDataSource` implementations with which collection sizes can be retrieved more efficiently than the full collection, `#var` can now be used as a short-hand notation for column name `R_rdf_sizeof_var`. +- Helpers have been added to export data from `RDataFrame` to RooFit datasets. See the "RooFit Libraries" section below for more details. +- The output format of `Display` has been significantly improved. + +### Experimental Distributed RDataFrame +The distributed RDataFrame module has been improved. Now it supports sending RDataFrame tasks to a [Dask](https://dask.org/) scheduler. Through Dask, RDataFrame can be also scaled to a cluster of machines managed through a batch system like HTCondor or Slurm. Here is an example: + +```python +import ROOT +from dask.distributed import Client +RDataFrame = ROOT.RDF.Experimental.Distributed.Dask.RDataFrame + +# In a Python script the Dask client needs to be initalized in a context +# Jupyter notebooks / Python session don't need this +if __name__ == "__main__": + client = Client("SCHEDULER_ADDRESS") + df = RDataFrame("mytree","myfile.root", daskclient=client) + # Proceed as usual + df.Define("x","someoperation").Histo1D("x") +``` + +Other notable additions and improvements include: + +- Greatly reduce distributed tasks processing overhead. This involved: + - Changing the distributed execution logic with the `TTree` data source to use `TEntryList` in order to select the range of entries that each task will read from the tree. This highly reduces the waiting time spent in retrieving the correct entries for processing, as it was previously done using the `Range` operation. + - Refactoring the internal mechanism to store information about data sources and create ranges for the distributed tasks accordingly. This will also allow in the future to easily extend the supported data sources in distributed RDataFrame. +- Refactor triggering of the computation graph in the distributed tasks, so that it now runs with the Python GIL released. This allows interoperability with frameworks like Dask that run different Python threads along the main processing one. +- Set minimum Python version to use this tool to 3.7. This allows using more modern Python functionality in distributed RDataFrame code and is in line with the Python support provided by Spark and Dask. +- Add support for the `DefinePerSample` operation. +- Fixed a bug that disregarded user provided `npartitions` parameter in distributed Spark execution. +- Improve support for friend trees in distributed executions with TTree as data source. This fixes a long-standing issue with distributed RDataFrame [#7584](https://github.com/root-project/root/issues/7584). +- Changed naming scheme of the files written by a distributed `Snapshot` operation. Now, each task will be assigned with a sequential id that will be appended to the name of the file created during the execution. For example, calling `Snapshot("mytree","myfile.root")` on a distributed RDataFrame with 3 partitions will create three files named like so: `myfile_0.root, myfile_1.root, myfile_2.root`. +- Add support for TChain data sources with no tree name and multiple different tree subnames. +- Raise an error if an in-memory-only TTree is passed as data source to a distributed RDataFrame. ### Other improvements @@ -78,7 +146,11 @@ The following people have contributed to this new version: ## Math Libraries +- `RVec` has been heavily re-engineered in order to add a small buffer optimization and to streamline its internals. The change should provide a small performance boost to + applications that make heavy use of `RVec`s and should otherwise be user-transparent. Please report any issues you should encounter. - I/O support of `RVec` objects has been optimized. As a side-effect, `RVec`s can now be read back as `std::vector`s and vice-versa. +- added `ROOT::VecOps::Drop`, an operation that removes `RVec` elements at the specified indices. +- handy aliases `ROOT::RVecI`, `ROOT::RVecD`, `ROOT::RVecF`, ..., have been introduced as short-hands for `RVec`, `RVec`, `RVec`, ... ## RooFit Libraries @@ -96,6 +168,48 @@ RooFit now contains two RDataFrame action helpers, `RooDataSetHelper` and `RooDa ``` For more details, consult the tutorial [rf408_RDataFrameToRooFit](https://root.cern/doc/v626/rf408__RDataFrameToRooFit_8C.html). +### Storing global observables in RooFit datasets + +RooFit groups model variables into *observables* and *parameters*, depending on if their values are stored in the dataset. +For fits with parameter constraints, there is a third kind of variables, called *global observables*. +These represent the results of auxiliary measurements that constrain the nuisance parameters. +In the RooFit implementation, a likelihood is generally the sum of two terms: + * the likelihood of the data given the parameters, where the normalization set is the set of observables (implemented by `RooNLLVar`) + * the constraint term, where the normalization set is the set of *global observables* (implemented by `RooConstraintSum`) + +Before this release, the global observable values were always taken from the model/pdf. +With this release, a mechanism is added to store a snapshot of global observables in any `RooDataSet` or `RooDataHist`. +For toy studies where the global observables assume a different values for each toy, the bookkeeping of the set of global observables and in particular their values is much easier with this change. + +Usage example for a model with global observables `g1` and `g2`: +```C++ +auto data = model.generate(x, 1000); // data has only the single observables x +data->setGlobalObservables(g1, g2); // now, data also stores a snapshot of g1 and g2 + +// If you fit the model to the data, the global observables and their values +// are taken from the dataset: +model.fitTo(*data); + +// You can still define the set of global observables yourself, but the values +// will be takes from the dataset if available: +model.fitTo(*data, GlobalObservables(g1, g2)); + +// To force `fitTo` to take the global observable values from the model even +// though they are in the dataset, you can use the new `GlobalObservablesSource` +// command argument: +model.fitTo(*data, GlobalObservables(g1, g2), GlobalObservablesSource("model")); +// The only other allowed value for `GlobalObservablesSource` is "data", which +// corresponds to the new default behavior explained above. +``` + +In case you create a RooFit dataset directly by calling its constructor, you can also pass the global observables in a command argument instead of calling `setGlobalObservables()` later: +```C++ +RooDataSet data{"dataset", "dataset", x, RooFit::GlobalObservables(g1, g2)}; +``` + +To access the set of global observables stored in a `RooAbsData`, call `RooAbsData::getGlobalObservables()`. +It returns a `nullptr` if no global observable snapshots are stored in the dataset. + ### Changes in `RooAbsPdf::fitTo` behaviour for multi-range fits The `RooAbsPdf::fitTo` and `RooAbsPdf::createNLL` functions accept a command argument to specify the fit range. @@ -106,7 +220,7 @@ From now on, the likelihoods are normalized by the sum of integrals in each rang ### Deprecation of the `RooMinuit` class -The `RooMinuit` class was the old interface between RooFit and minuit. With ROOT version 5.24, a the more general `RooMinimizer` adapter was introduced, which became the default with ROOT 6.08. +The `RooMinuit` class was the old interface between RooFit and minuit. With ROOT version 5.24, the more general `RooMinimizer` adapter was introduced, which became the default with ROOT 6.08. Before 6.26, it was possible to still use the `RooMinuit` by passing the `Minimizer("OldMinuit", "minimizer")` command argument to `RooAbsPdf::fitTo()`. This option is now removed. @@ -118,6 +232,8 @@ The class version of `RooAbsArg` was incremented from 7 to 8 in this release. In - Implement the option `X+` and `Y+` for reverse axis on TGraph. - Offsets for axis titles with absolute-sized fonts (size%10 == 3) are now relative only to the font size (i.e. no longer relative to pad dimensions). +- In `TPaletteAxis` when the palette width is bigger than the palette height, the palette + in automatically drawn horizontally. ## 3D Graphics Libraries @@ -167,3 +283,8 @@ The class version of `RooAbsArg` was incremented from 7 to 8 in this release. In ## Build, Configuration and Testing Infrastructure + +## PyROOT + +- The `ROOT` Python module is now properly serializable so that it is automatically available in the Python environment if a function or ROOT object needs to be serialized. See issue [#6764](https://github.com/root-project/root/issues/6764) for a concrete usecase. +- Improve overload resolution of functions that accept classes with long inheritance trees. Now prefer to call the function overload of the most derived class type (PR [#9092](https://github.com/root-project/root/pull/9092)). \ No newline at end of file diff --git a/bindings/CMakeLists.txt b/bindings/CMakeLists.txt index 46989ba7dd8e2..ba96f41142784 100644 --- a/bindings/CMakeLists.txt +++ b/bindings/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. +# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. # All rights reserved. # # For the licensing terms see $ROOTSYS/LICENSE. @@ -25,7 +25,8 @@ if(pyroot) add_subdirectory(tpython) add_subdirectory(jupyroot) add_subdirectory(jsmva) - if(dataframe) + if(dataframe AND (PYTHON_VERSION_STRING_Development_Main VERSION_GREATER_EQUAL 3.7)) + # TODO: Add a status message to signal both if DistRDF is being built or if it isn't add_subdirectory(experimental/distrdf) endif() diff --git a/bindings/experimental/distrdf/CMakeLists.txt b/bindings/experimental/distrdf/CMakeLists.txt index 6aef05fb1b25e..623f1b8f7ce21 100644 --- a/bindings/experimental/distrdf/CMakeLists.txt +++ b/bindings/experimental/distrdf/CMakeLists.txt @@ -23,17 +23,16 @@ set(py_sources DistRDF/Backends/Utils.py DistRDF/Backends/Spark/__init__.py DistRDF/Backends/Spark/Backend.py + DistRDF/Backends/AWS/__init__.py + DistRDF/Backends/AWS/Backend.py ) -foreach(val RANGE ${how_many_pythons}) - list(GET python_executables ${val} python_executable) - - # Compile .py files - foreach(py_source ${py_sources}) - install(CODE "execute_process(COMMAND ${python_executable} -m py_compile ${localruntimedir}/${py_source})") - install(CODE "execute_process(COMMAND ${python_executable} -O -m py_compile ${localruntimedir}/${py_source})") - endforeach() - +# Compile .py files +# We include DistRDF in the build only if Python 3.7+ is used, +# so we can directly use the main Python executable to compile the sources +foreach(py_source ${py_sources}) + install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE_Development_Main} -m py_compile ${localruntimedir}/${py_source})") + install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE_Development_Main} -O -m py_compile ${localruntimedir}/${py_source})") endforeach() # Install Python sources and bytecode diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/AWS_utils.py b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/AWS_utils.py new file mode 100644 index 0000000000000..9a6a732a110ce --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/AWS_utils.py @@ -0,0 +1,167 @@ +import logging +import json +import base64 +import time +import cloudpickle as pickle +import boto3 +import botocore +import sys +import os +from typing import Optional +from pathlib import Path + + +class AWSServiceWrapper: + INVOCATION_RETRIALS_COUNT = 1 # 3 + + def __init__(self, region): + self.region = region + + def invoke_root_lambda(self, + root_range, + script, + certs, + headers, + bucket_name, + logger=logging.getLogger() + ) -> Optional[str]: + """ + Invoke root lambda. + Args: + root_range (Range): Range of data. + script (function): A function that performs an operation on + a range of data. + logger (logging.Logger): + Returns: + Optional[str]: Name of file created by lambda or None if procedure + has failed. + """ + + trials = self.INVOCATION_RETRIALS_COUNT + config = botocore.config.Config(retries={'total_max_attempts': 1}, + read_timeout=900, + connect_timeout=900 + ) + client = boto3.client('lambda', region_name=self.region, config=config) + + payload = json.dumps({ + 'range': self.encode_object(root_range), + 'script': script, + 'start': str(root_range.start), + 'end': str(root_range.end), + 'filelist': str(root_range.filelist), + 'friend_info': self.encode_object(root_range.friend_info), + 'cert': str(base64.b64encode(certs)), + 'headers': headers + }) + + # Maybe here give info about number of invoked lambda for awsmonitor + + filename: Optional[str] = None + + while trials > 0: + trials -= 1 + try: + response = client.invoke( + FunctionName='root_lambda', + InvocationType='RequestResponse', + Payload=bytes(payload, encoding='utf8') + ) + + payload = self.get_response_payload(response) + + if 'FunctionError' in response or payload.get('statusCode') == 500: + exception, msg = self.process_lambda_error(payload) + raise exception(msg) + + filename = json.loads(payload.get('filename', 'null')) + + except botocore.exceptions.ClientError as error: + # AWS site errors + logger.warning(error) + logger.warning(error['Error']['Message']) + except Exception as error: + # All other errors + logger.warning(str(error) + " (" + type(error).__name__ + ")") + else: + while not self.s3_object_exists(bucket_name, filename): + time.sleep(0.5) + break + + time.sleep(1) + + return filename + + @staticmethod + def get_response_payload(response): + try: + return json.loads(response['Payload'].read()) + except Exception: + return {} + + @staticmethod + def process_lambda_error(payload): + try: + # Get error specification and remove additional + # quotas (side effect of serialization) + error_type = payload['errorType'][1:-1] + error_message = payload['errorMessage'][1:-1] + exception = getattr(sys.modules['builtins'], error_type) + msg = f"Lambda raised an exception: {error_message}" + except Exception: + exception = RuntimeError + msg = (f"Lambda raised an exception: (type={payload['errorType']}," + f"message={payload['errorMessage']})") + return exception, msg + + def get_partial_result_from_s3(self, filename, bucket_name): + pickled_file = self.get_file_content_from_s3(filename, bucket_name) + return pickle.loads(pickled_file) + + def get_file_content_from_s3(self, filename, bucket_name): + s3_client = boto3.client('s3', region_name=self.region) + response = s3_client.get_object(Bucket=bucket_name, Key=filename) + return response['Body'].read() + + def clean_s3_bucket(self, bucket_name): + s3_resource = boto3.resource('s3', region_name=self.region) + s3_bucket = s3_resource.Bucket(name=bucket_name) + s3_bucket.objects.all().delete() + + def s3_object_exists(self, bucket_name, filename): + try: + s3_client = boto3.client('s3', region_name=self.region) + s3_client.head_object(Bucket=bucket_name, Key=filename) + return True + except botocore.exceptions.ClientError: + return False + + def get_ssm_parameter_value(self, name): + ssm_client = boto3.client('ssm', region_name=self.region) + param = ssm_client.get_parameter(Name=name) + return param['Parameter']['Value'] + + @staticmethod + def encode_object(object_to_encode) -> str: + return str(base64.b64encode(pickle.dumps(object_to_encode))) + + def get_from_s3(self, filename, bucket_name, directory): + s3_client = boto3.client('s3', region_name=self.region) + local_filename = os.path.join(directory, filename) + s3_client.download_file(bucket_name, filename, local_filename) + + # tfile = ROOT.TFile(local_filename, 'OPEN') + # result = [] + + # Get all objects from TFile + # for key in tfile.GetListOfKeys(): + # result.append(key.ReadObj()) + # result[-1].SetDirectory(0) + # tfile.Close() + with open(local_filename, 'rb') as pickle_file: + result = pickle.load(pickle_file) + + # Remove temporary root file + Path(local_filename).unlink() + + return result diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/Backend.py b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/Backend.py new file mode 100644 index 0000000000000..c45a2da8cd439 --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/Backend.py @@ -0,0 +1,157 @@ +from __future__ import print_function + +import logging +import functools +import time + +from DistRDF import DataFrame +from DistRDF import Node +from DistRDF.Backends import Base + +from .flushing_logger import FlushingLogger +from .AWS_utils import AWSServiceWrapper +from .reducer import Reducer +from concurrent.futures import ThreadPoolExecutor + + +class AWS(Base.BaseBackend): + """ + Backend that executes the computational graph using using AWS Lambda + for distributed execution. + """ + + MIN_NPARTITIONS = 8 + npartitions = 32 + TOKEN_PATH = '/tmp/certs' + + def __init__(self, config={}): + """ + Config for AWS is same as in Dist backend, + more support will be added in future. + """ + super(AWS, self).__init__() + self.logger = FlushingLogger() if logging.root.level >= logging.INFO else logging.getLogger() + self.npartitions = self._get_partitions() + self.region = config.get('region') or 'us-east-1' + self.paths = [] + self.aws_service_wrapper = AWSServiceWrapper(self.region) + + def _get_partitions(self): + return int(self.npartitions or AWS.MIN_NPARTITIONS) + + def make_dataframe(self, *args, **kwargs): + """ + Creates an instance of distributed RDataFrame that can send computations + to a Spark cluster. + """ + # Set the number of partitions for this dataframe, one of the following: + # 1. User-supplied `npartitions` optional argument + # 2. An educated guess according to the backend, using the backend's + # `optimize_npartitions` function + # 3. Set `npartitions` to 2 + headnode = Node.HeadNode(*args) + return DataFrame.RDataFrame(headnode, self, **kwargs) + + def ProcessAndMerge(self, ranges, mapper, reducer): + """ + Performs map-reduce using AWS Lambda. + Args: + mapper (function): A function that runs the computational graph + and returns a list of values. + reducer (function): A function that merges two lists that were + returned by the mapper. + Returns: + list: A list representing the values of action nodes returned + after computation (Map-Reduce). + """ + + invoke_func, download_func, processing_bucket = self.create_init_arguments(mapper) + + self.logger.info(f'Before lambdas invoke. Number of lambdas: {len(ranges)}') + + invoke_begin = time.time() + + files = self.invoke_and_download(invoke_func, download_func, ranges) + + reduce_begin = time.time() + + result = Reducer.tree_reduce(reducer, files) + + bench = ( + len(ranges), + reduce_begin - invoke_begin, + time.time() - reduce_begin + ) + + # Clean up intermediate objects after we're done + self.aws_service_wrapper.clean_s3_bucket(processing_bucket) + + print(bench) + + return result + + def create_init_arguments(self, mapper): + """ + Create init arguments for ProcessAndMerge method. + """ + + pickled_mapper = AWSServiceWrapper.encode_object(mapper) + pickled_headers = AWSServiceWrapper.encode_object(self.paths) + processing_bucket = self.aws_service_wrapper.get_ssm_parameter_value('processing_bucket') + + try: + with open(self.TOKEN_PATH, "rb") as f: + certs = f.read() + except FileNotFoundError: + certs = b'' + + invoke_lambda = functools.partial( + self.aws_service_wrapper.invoke_root_lambda, + script=pickled_mapper, + certs=certs, + headers=pickled_headers, + bucket_name=processing_bucket, + logger=self.logger) + + download_partial = functools.partial( + self.aws_service_wrapper.get_partial_result_from_s3, + bucket_name=processing_bucket) + + return invoke_lambda, download_partial, processing_bucket + + def invoke_and_download(self, invoke, download, ranges): + files = [] + + def download_callback(future): + filename = future.result() + if filename is not None: + files.append(download(filename)) + + with ThreadPoolExecutor(max_workers=len(ranges)) as executor: + for root_range in ranges: + future = executor.submit(invoke, root_range) + future.add_done_callback(download_callback) + + if len(files) < len(ranges): + raise Exception(f'Some lambdas failed after multiple retrials') + + return files + + def distribute_unique_paths(self, paths): + """ + Spark supports sending files to the executors via the + `SparkContext.addFile` method. This method receives in input the path + to the file (relative to the path of the current python session). The + file is initially added to the Spark driver and then sent to the + workers when they are initialized. + + Args: + paths (set): A set of paths to files that should be sent to the + distributed workers. + """ + pass + + def add_header(self, path: str): + with open(path, 'r') as f: + contents = f.read() + self.paths.append((path, contents)) diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/__init__.py b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/__init__.py new file mode 100644 index 0000000000000..8a0bea05ee256 --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/__init__.py @@ -0,0 +1,21 @@ +## @author Vincenzo Eduardo Padulano +# @author Enric Tejedor +# @date 2021-02 + +################################################################################ +# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +def RDataFrame(*args, **kwargs): + """ + Create an RDataFrame object that can run computations on a Spark cluster. + """ + + from DistRDF.Backends.AWS import Backend + aws = Backend.AWS() + + return aws.make_dataframe(*args, **kwargs) diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/flushing_logger.py b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/flushing_logger.py new file mode 100644 index 0000000000000..d8d12a037dd54 --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/flushing_logger.py @@ -0,0 +1,28 @@ +import logging + + +class FlushingLogger: + FLUSHED_METHODS = [ + 'info', + 'warning', + 'debug', + 'error', + 'critical' + ] + + def __init__(self): + self.logger = logging.getLogger() + + def __getattr__(self, name): + method = getattr(self.logger, name) + + if name not in self.FLUSHED_METHODS: + return method + + def flushed_method(msg, *args, **kwargs): + method(msg, *args, **kwargs) + for h in self.logger.handlers: + h.flush() + + return flushed_method + diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/reducer.py b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/reducer.py new file mode 100644 index 0000000000000..48e5627663bab --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/AWS/reducer.py @@ -0,0 +1,67 @@ +from concurrent.futures import ThreadPoolExecutor + + +class Reducer: + + @staticmethod + def tree_reduce(reducer, iterable, chunk_size=2, min_size=4): + """ + Parallel tree reduction. + At each step objects to reduce are divided into + groups (chunks) and each chunk is then reduced to one + object in parallel. Whole process lasts until there is + at most min_size objects, which are then reduced + sequentially. + """ + + to_process = iterable + + while len(to_process) > min_size: + chunks = Reducer.divide_into_chunks(to_process, chunk_size=chunk_size) + to_process = Reducer.parallel_reduce(chunks, reducer) + + return Reducer.reduce(reducer, to_process) + + @staticmethod + def divide_into_chunks(iterable, chunk_size=2): + """ + Divide list into chunks of given size. + If even division is impossible, leftovers are put in the last entry. + + Returns: + list: List of tuples each of size chunk_size. + + >>> r = Reducer() + >>> r.divide_into_chunks([1, 2, 3, 4, 5]) + [(1, 2), (3, 4), (5,)] + >>> r.divide_into_chunks([1, 2, 3, 4, 5, 6], chunk_size=3) + [(1, 2, 3), (4, 5, 6)] + >>> r.divide_into_chunks([], chunk_size=1) + [] + + """ + + if chunk_size <= 0: + return [] + + return [tuple(iterable[i:i+chunk_size]) + for i in range(0, len(iterable), chunk_size)] + + @staticmethod + def parallel_reduce(chunks, reducer): + with ThreadPoolExecutor(len(chunks)) as executor: + futures = [executor.submit(Reducer.reduce, reducer, chunk) + for chunk in chunks] + results = [future.result() for future in futures] + return results + + @staticmethod + def reduce(reducer, iterable): + if not iterable: + return None + + acc = iterable[0] + for i in range(1, len(iterable)): + acc = reducer(acc, iterable[i]) + + return acc diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/Base.py b/bindings/experimental/distrdf/python/DistRDF/Backends/Base.py index 1cd33941399af..3ebec238a3e6d 100644 --- a/bindings/experimental/distrdf/python/DistRDF/Backends/Base.py +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/Base.py @@ -15,6 +15,7 @@ from abc import abstractmethod import ROOT + from DistRDF.Backends import Utils from DistRDF.HeadNode import TreeHeadNode @@ -43,6 +44,7 @@ class BaseBackend(ABC): "AsNumpy", "Count", "Define", + "DefinePerSample", "Fill", "Filter", "Graph", @@ -117,9 +119,15 @@ def execute(self, generator): given RDataFrame object and a range of entries. The range is needed for the `Snapshot` operation. """ - headnode = generator.headnode - computation_graph_callable = generator.get_callable() + # Check if the workflow must be generated in optimized mode + optimized = ROOT.RDF.Experimental.Distributed.optimized + + if optimized: + computation_graph_callable = generator.get_callable_optimized() + else: + computation_graph_callable = generator.get_callable() + headnode = generator.headnode if isinstance(headnode, TreeHeadNode): maintreename = headnode.maintreename defaultbranches = headnode.defaultbranches @@ -149,7 +157,15 @@ def mapper(current_range): list: This respresents the list of (mergeable)values of all action nodes in the computational graph. """ - import ROOT + # Disable graphics functionality in ROOT. It is not needed inside a + # distributed task + ROOT.gROOT.SetBatch(True) + # Enable thread safety for the whole mapper function. We need to do + # this since two tasks could be invoking the C++ interpreter + # simultaneously, given that this function will release the GIL + # before calling into C++ to run the event loop. Dask multi-threaded + # or even multi-process workers could trigger such a scenario. + ROOT.EnableThreadSafety() # We have to decide whether to do this in Dist or in subclasses # Utils.declare_headers(worker_includes) # Declare headers if any @@ -228,16 +244,26 @@ def mapper(current_range): # user, then limit processing to the entries in this range. rdf = ROOT.RDataFrame(nentries).Range(current_range.start, current_range.end) + if optimized: + # Create the RDF computation graph and execute it on this ranged + # dataset. The results of the actions of the graph and their types + # are returned + results, res_types = computation_graph_callable(rdf, current_range.id) + + # Get RResultPtrs out of the type-erased RResultHandles by + # instantiating with the type of the value + mergeables = [ + ROOT.ROOT.Detail.RDF.GetMergeableValue(res.GetResultPtr[res_type]()) + if isinstance(res, ROOT.RDF.RResultHandle) + else res + for res, res_type in zip(results, res_types) + ] + else: + # Output of the callable + resultptr_list = computation_graph_callable(rdf, current_range.id) - # Output of the callable - resultptr_list = computation_graph_callable(rdf, current_range.id) + mergeables = [Utils.get_mergeablevalue(resultptr) for resultptr in resultptr_list] - mergeables = [ - resultptr # Here resultptr is already the result value - if isinstance(resultptr, (dict, list)) - else ROOT.ROOT.Detail.RDF.GetMergeableValue(resultptr) - for resultptr in resultptr_list - ] return mergeables def reducer(mergeables_out, mergeables_in): @@ -259,36 +285,8 @@ def reducer(mergeables_out, mergeables_in): list: The list of updated (mergeable)values. """ - import ROOT - - # We still need the list index to modify results of `Snapshot` and - # `AsNumpy` in place. - for index, (mergeable_out, mergeable_in) in enumerate( - zip(mergeables_out, mergeables_in)): - # Create a global list with all the files of the partial - # snapshots. - if isinstance(mergeable_out, list): - mergeables_out[index].extend(mergeable_in) - - # Concatenate the partial numpy arrays along the same key of - # the dictionary. - elif isinstance(mergeable_out, dict): - # Import numpy lazily - try: - import numpy - except ImportError: - raise ImportError("Failed to import numpy during distributed RDataFrame reduce step.") - mergeables_out[index] = { - key: numpy.concatenate([mergeable_out[key], - mergeable_in[key]]) - for key in mergeable_out - } - - # The `MergeValues` function modifies the arguments in place - # so there's no need to access the list elements. - else: - ROOT.ROOT.Detail.RDF.MergeValues( - mergeable_out, mergeable_in) + for mergeable_out, mergeable_in in zip(mergeables_out, mergeables_in): + Utils.merge_values(mergeable_out, mergeable_in) return mergeables_out @@ -300,16 +298,9 @@ def reducer(mergeables_out, mergeables_in): # Set the value of every action node for node, value in zip(nodes, values): if node.operation.name == "Snapshot": - # Retrieve treename from operation args and start TChain - snapshot_treename = node.operation.args[0] - snapshot_chain = ROOT.TChain(snapshot_treename) - # Add partial snapshot files to the chain - for filename in value: - snapshot_chain.Add(filename) - # Create a new rdf with the chain and return that to user - node.value = self.make_dataframe(snapshot_chain) - elif node.operation.name == "AsNumpy": - node.value = value + # Retrieving a new distributed RDataFrame from the result of a + # distributed Snapshot needs knowledge of the correct backend + node.value = value.GetValue(self) else: node.value = value.GetValue() diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/Dask/Backend.py b/bindings/experimental/distrdf/python/DistRDF/Backends/Dask/Backend.py new file mode 100644 index 0000000000000..74c6ca18e2cfc --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/Dask/Backend.py @@ -0,0 +1,155 @@ +# @author Vincenzo Eduardo Padulano +# @author Enric Tejedor +# @date 2021-11 + +################################################################################ +# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ +import os + +from DistRDF import DataFrame +from DistRDF import HeadNode +from DistRDF.Backends import Base +from DistRDF.Backends import Utils + +try: + import dask + from dask.distributed import Client, LocalCluster, progress, get_worker +except ImportError: + raise ImportError(("cannot import a Dask component. Refer to the Dask documentation " + "for installation instructions.")) + + +class DaskBackend(Base.BaseBackend): + """Dask backend for distributed RDataFrame.""" + + def __init__(self, daskclient=None): + super(DaskBackend, self).__init__() + # If the user didn't explicitly pass a Client instance, the argument + # `daskclient` will be `None`. In this case, we create a default Dask + # client connected to a cluster instance with N worker processes, where + # N is the number of cores on the local machine. + self.client = (daskclient if daskclient is not None else + Client(LocalCluster(n_workers=os.cpu_count(), threads_per_worker=1, processes=True))) + + def optimize_npartitions(self): + """ + Attempts to compute a clever number of partitions for the current + execution. Currently, we try to get the total number of worker logical + cores in the cluster. + """ + try: + return sum(worker['nthreads'] for worker in self.client.scheduler_info()['workers'].values()) + except KeyError: + # If the scheduler doesn't have some information about the workers + return self.MIN_NPARTITIONS + + def ProcessAndMerge(self, ranges, mapper, reducer): + """ + Performs map-reduce using Dask framework. + + Args: + mapper (function): A function that runs the computational graph + and returns a list of values. + + reducer (function): A function that merges two lists that were + returned by the mapper. + + Returns: + list: A list representing the values of action nodes returned + after computation (Map-Reduce). + """ + + # These need to be passed as variables, because passing `self` inside + # following `dask_mapper` function would trigger serialization errors + # like the following: + # + # AttributeError: Can't pickle local object 'DaskBackend.ProcessAndMerge..dask_mapper' + # TypeError: cannot pickle '_asyncio.Task' object + # + # Which boil down to the self.client object not being serializable + headers = self.headers + shared_libraries = self.shared_libraries + + def dask_mapper(current_range): + """ + Gets the paths to the file(s) in the current executor, then + declares the headers found. + + Args: + current_range (tuple): The current range of the dataset being + processed on the executor. + + Returns: + function: The map function to be executed on each executor, + complete with all headers needed for the analysis. + """ + # Retrieve the current worker local directory + localdir = get_worker().local_directory + + # Get and declare headers on each worker + headers_on_executor = [ + os.path.join(localdir, os.path.basename(filepath)) + for filepath in headers + ] + Utils.declare_headers(headers_on_executor) + + # Get and declare shared libraries on each worker + shared_libs_on_ex = [ + os.path.join(localdir, os.path.basename(filepath)) + for filepath in shared_libraries + ] + Utils.declare_shared_libraries(shared_libs_on_ex) + + return mapper(current_range) + + dmapper = dask.delayed(dask_mapper) + dreducer = dask.delayed(reducer) + + mergeables_lists = [dmapper(range) for range in ranges] + + while len(mergeables_lists) > 1: + mergeables_lists.append( + dreducer(mergeables_lists.pop(0), mergeables_lists.pop(0))) + + # Here we start the progressbar for the current RDF computation graph + # running on the Dask client. This expects a future object, so we need + # convert the last delayed object from the list above to a future + # through the `persist` call. This also starts the computation in the + # background, but the time difference is negligible. The progressbar is + # properly shown in the terminal, whereas in the notebook it can be + # shown only if it's the last call in a cell. Since we're encapsulating + # it in this class, it won't be shown. Full details at + # https://docs.dask.org/en/latest/diagnostics-distributed.html#dask.distributed.progress + final_results = mergeables_lists.pop().persist() + progress(final_results) + + return final_results.compute() + + def distribute_unique_paths(self, paths): + """ + Dask supports sending files to the workes via the `Client.upload_file` + method. Its stated purpose is to send local Python packages to the + nodes, but in practice it uploads the file to the path stored in the + `local_directory` attribute of each worker. + """ + for filepath in paths: + self.client.upload_file(filepath) + + def make_dataframe(self, *args, **kwargs): + """ + Creates an instance of distributed RDataFrame that can send computations + to a Dask cluster. + """ + # Set the number of partitions for this dataframe, one of the following: + # 1. User-supplied `npartitions` optional argument + # 2. An educated guess according to the backend, using the backend's + # `optimize_npartitions` function + # 3. Set `npartitions` to 2 + npartitions = kwargs.pop("npartitions", self.optimize_npartitions()) + headnode = HeadNode.get_headnode(npartitions, *args) + return DataFrame.RDataFrame(headnode, self) diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/Dask/__init__.py b/bindings/experimental/distrdf/python/DistRDF/Backends/Dask/__init__.py new file mode 100644 index 0000000000000..a5c74c4c2745d --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/Dask/__init__.py @@ -0,0 +1,22 @@ +# @author Vincenzo Eduardo Padulano +# @author Enric Tejedor +# @date 2021-11 + +################################################################################ +# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +def RDataFrame(*args, **kwargs): + """ + Create an RDataFrame object that can run computations on a Dask cluster. + """ + + from DistRDF.Backends.Dask import Backend + daskclient = kwargs.get("daskclient", None) + daskbackend = Backend.DaskBackend(daskclient=daskclient) + + return daskbackend.make_dataframe(*args, **kwargs) diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/Spark/Backend.py b/bindings/experimental/distrdf/python/DistRDF/Backends/Spark/Backend.py index b0db81f510d37..2eb6cc93d07d3 100644 --- a/bindings/experimental/distrdf/python/DistRDF/Backends/Spark/Backend.py +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/Spark/Backend.py @@ -65,15 +65,15 @@ def __init__(self, sparkcontext=None): self.sc = pyspark.SparkContext.getOrCreate() def optimize_npartitions(self): - numex = self.sc.getConf().get("spark.executor.instances") - numcoresperex = self.sc.getConf().get("spark.executor.cores") - - if numex is not None: - if numcoresperex is not None: - return int(numex) * int(numcoresperex) - return int(numex) - else: - return self.MIN_NPARTITIONS + """ + The SparkContext.defaultParallelism property roughly translates to the + available amount of logical cores on the cluster. Some examples: + - spark.master = local[n]: returns n. + - spark.executor.instances = m and spark.executor.cores = n: returns `n*m`. + By default, the minimum number this returns is 2 if the context + doesn't know any better. For example, if dynamic allocation is enabled. + """ + return self.sc.defaultParallelism def ProcessAndMerge(self, ranges, mapper, reducer): """ diff --git a/bindings/experimental/distrdf/python/DistRDF/Backends/Utils.py b/bindings/experimental/distrdf/python/DistRDF/Backends/Utils.py index 147a722faf6ec..6f493ad8bd34c 100644 --- a/bindings/experimental/distrdf/python/DistRDF/Backends/Utils.py +++ b/bindings/experimental/distrdf/python/DistRDF/Backends/Utils.py @@ -1,4 +1,4 @@ -## @author Vincenzo Eduardo Padulano +# @author Vincenzo Eduardo Padulano # @author Enric Tejedor # @date 2021-02 @@ -13,7 +13,12 @@ import logging import os +from functools import singledispatch + import ROOT +from ROOT.pythonization._rdataframe import AsNumpyResult + +from DistRDF.PythonMergeables import SnapshotResult logger = logging.getLogger(__name__) @@ -145,3 +150,57 @@ def check_pcm_in_library_path(shared_library_path): } return pcm_paths, libraries_path + + +@singledispatch +def get_mergeablevalue(resultptr): + """ + Generally the input argument to this function is an RResultPtr, for which a + corresponding RMergeableValue type already exists. Call into the C++ + function to handle this case. + """ + return ROOT.Detail.RDF.GetMergeableValue(resultptr) + + +@get_mergeablevalue.register(AsNumpyResult) +def _(resultptr): + """ + Results coming from an `AsNumpy` operation can be merged with others, but + we need to make sure to call its `GetValue` method since that will populate + the private attribute `_py_arrays` (which is the actual dictionary of + numpy arrays extracted from the RDataFrame columns). This extra call is an + insurance against backends that do not automatically serialize objects + returned by the mapper function (otherwise this would be taken care by the + `AsNumpyResult`'s `__getstate__` method). + """ + resultptr.GetValue() + return resultptr + + +@get_mergeablevalue.register(SnapshotResult) +def _(resultptr): + """ + When performing a distributed Snapshot we return an object holding the name + of the dataset and the path to the partial snapshot. We can directly return + the object, no extra work needed. + """ + return resultptr + + +@singledispatch +def merge_values(mergeable_out, mergeable_in): + """ + Generally the arguments are `RMergeableValue` instances that can be directly + passed to the C++ function responsible for merging them. + """ + ROOT.Detail.RDF.MergeValues(mergeable_out, mergeable_in) + + +@merge_values.register(AsNumpyResult) +@merge_values.register(SnapshotResult) +def _(mergeable_out, mergeable_in): + """ + Mergeables coming from `Snapshot` or `AsNumpy` operations have their own + `Merge` method. + """ + mergeable_out.Merge(mergeable_in) diff --git a/bindings/experimental/distrdf/python/DistRDF/ComputationGraphGenerator.py b/bindings/experimental/distrdf/python/DistRDF/ComputationGraphGenerator.py index fb28e5ac6f612..82c161e164a41 100644 --- a/bindings/experimental/distrdf/python/DistRDF/ComputationGraphGenerator.py +++ b/bindings/experimental/distrdf/python/DistRDF/ComputationGraphGenerator.py @@ -1,4 +1,4 @@ -## @author Vincenzo Eduardo Padulano +# @author Vincenzo Eduardo Padulano # @author Enric Tejedor # @date 2021-02 @@ -10,6 +10,16 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ +import logging + +import ROOT + +from DistRDF.CppWorkflow import CppWorkflow +from DistRDF.PythonMergeables import SnapshotResult + +logger = logging.getLogger(__name__) + + class ComputationGraphGenerator(object): """ Class that generates a callable to parse a DistRDF graph. @@ -60,9 +70,162 @@ def get_action_nodes(self, node_py=None): return return_nodes + def _make_op_lazy_if_needed(self, operation, range_id): + """ + We may need to change the attributes of some operations (currently + Snapshot and AsNumpy), to make them lazy before triggering + the computation graph. + """ + + if operation.name == "Snapshot": + # Retrieve filename and append range boundaries + filename = operation.args[1].partition(".root")[0] + path_with_range = "{}_{}.root".format(filename, range_id) + # Create a partial snapshot on the current range + operation.args[1] = path_with_range + + if len(operation.args) == 2: + # Only the first two mandatory arguments were passed + # Only the following overload is possible + # Snapshot(std::string_view treename, std::string_view filename, std::string_view columnNameRegexp = "") + operation.args.append("") # Append empty regex + + if len(operation.args) == 4: + # An RSnapshotOptions instance was passed as fourth argument + # Make it lazy and keep the other options + operation.args[3].fLazy = True + else: + # We already appended an empty regex for the 2 mandatory arguments overload + # All other overloads have 3 mandatory arguments + # We just need to append a lazy RSnapshotOptions now + lazy_options = ROOT.RDF.RSnapshotOptions() + lazy_options.fLazy = True + operation.args.append(lazy_options) # Append RSnapshotOptions + elif operation.name == "AsNumpy": + operation.kwargs["lazy"] = True # Make it lazy + + def generate_computation_graph(self, previous_node, range_id, distrdf_node=None): + """ + Generates the RDF computation graph by recursively retrieving + information from the DistRDF nodes. + + Args: + previous_node (Any): The node in the RDF computation graph on which + the operation of the current recursive state is called. In the + first recursive state, this corresponds to the RDataFrame + object that will be processed. Specifically, if the head node + of the computation graph is an EmptySourceHeadNode, then the + first current node will actually be the result of a call to the + Range operation. If the head node is a TreeHeadNode then the + node will be an actual RDataFrame. Successive recursive states + will receive the result of an RDF operation call + (e.g. Histo1D, Count). + range_id (int): The id of the current range. Needed to assign a + file name to a partial Snapshot if it was requested. + distrdf_node (DistRDF.Node.Node | None): The current DistRDF node in + the computation graph. In the first recursive state this is None + and it will be set equal to the DistRDF headnode. + + Returns: + list: List of actions of the computation graph to be triggered. Each + element is some kind of promise of a result (usually an + RResultPtr). Exceptions are the 'AsNumpy' operation for which an + 'AsNumpyResult' is returned and the 'Snapshot' operation for which a + 'SnapshotResult' is returned. + """ + future_results = [] + + if distrdf_node is None: + # In the first recursive state, just set the + # current DistRDF node as the head node + distrdf_node = self.headnode + else: + # Execute the current operation using the output of the previous + # node + RDFOperation = getattr(previous_node, distrdf_node.operation.name) + operation = distrdf_node.operation + self._make_op_lazy_if_needed(operation, range_id) + pyroot_node = RDFOperation(*operation.args, **operation.kwargs) + + # The result is a pyroot object which is stored together with + # the DistRDF node. This binds the pyroot object lifetime to the + # DistRDF node, so both nodes will be kept alive as long as there + # is a valid reference pointing to the DistRDF node. + distrdf_node.pyroot_node = pyroot_node + + # Set the next `previous_node` input argument to the `pyroot_node` + # we just retrieved + previous_node = pyroot_node + + if (operation.is_action() or operation.is_instant_action()): + if operation.name == "Snapshot": + future_results.append(SnapshotResult(operation.args[0], [operation.args[1]])) + else: + future_results.append(pyroot_node) + + for child_node in distrdf_node.children: + # Recurse through children and get their output + prev_results = self.generate_computation_graph(previous_node, range_id, child_node) + + # Attach the output of the children node + future_results.extend(prev_results) + + return future_results + + def trigger_computation_graph(self, starting_node, range_id): + """ + Trigger the computation graph. + + The list of actions to be performed is retrieved by calling + generate_computation_graph. Afterwards, the C++ RDF computation graph is + triggered through the `ROOT::Internal::RDF::TriggerRun` function with + the GIL released. + + Args: + starting_node (ROOT.RDF.RNode): The node where the generation of the + computation graph is started. Either an actual RDataFrame or the + result of a Range operation (in case of empty data source). + range_id (int): The id of the current range. Needed to assign a + file name to a partial Snapshot if it was requested. + + Returns: + list: A list of objects that can be either used as or converted into + mergeable values. + """ + actions = self.generate_computation_graph(starting_node, range_id) + + # Trigger computation graph with the GIL released + rnode = ROOT.RDF.AsRNode(starting_node) + ROOT.Internal.RDF.TriggerRun.__release_gil__ = True + ROOT.Internal.RDF.TriggerRun(rnode) + + # Return a list of objects that can be later merged. In most cases this + # is still made of RResultPtrs that will then be used as input arguments + # to `ROOT::RDF::Detail::GetMergeableValue`. For `AsNumpy`, it returns + # an instance of `AsNumpyResult`. For `Snapshot`, it returns a + # `SnapshotResult` + return actions + def get_callable(self): + """ + Prunes the DistRDF computation graph from unneeded nodes and returns + a function responsible for creating and triggering the corresponding + C++ RDF computation graph. + """ + # Prune the graph to check user references + # This needs to be done at this point, on the client machine, since the + # `trigger_computation_graph` will be called inside a distributed worker. + # Doing the pruning here makes sure we do it only once and not waste + # extra time on the remote machines. + self.headnode.graph_prune() + + return self.trigger_computation_graph + + def get_callable_optimized(self): """ Converts a given graph into a callable and returns the same. + The callable is optimized to execute the graph with compiled C++ + performance. Returns: function: The callable that takes in a PyROOT RDataFrame object @@ -72,74 +235,49 @@ def get_callable(self): # Prune the graph to check user references self.headnode.graph_prune() - def generate_computation_graph(node_cpp, range_id, node_py=None): + def run_computation_graph(rdf_node, range_id): """ - The callable that recurses through the DistRDF nodes and executes - operations from a starting (PyROOT) RDF node. + The callable that traverses the DistRDF graph nodes, generates the + code to create the same graph in C++, compiles it and runs it. + This function triggers the event loop via the CppWorkflow class. Args: - node_cpp (ROOT.RDF.RNode): The current state's ROOT CPP node. - Initially this is the PyROOT RDataFrame object. - range_id (int): The id of the current range. Needed to assign a - file name to a partial Snapshot if it was requested. - node_py (optional): The current state's DistRDF node. If `None`, - it takes the value of `self.headnode`. + rdf_node (ROOT.RDF.RNode): The RDataFrame node that will serve as + the root of the computation graph. + range_id (int): Id of the current range. Needed to assign a name + to a partial Snapshot output file. Returns: - list: A list of :obj:`ROOT.RResultPtr` objects in DFS order of - their corresponding actions in the graph. + tuple[list, list]: the first element is the list of results of the actions + in the C++ workflow, the second element is the list of + result types corresponding to those actions. """ - return_vals = [] - parent_node = node_cpp + # Generate the code of the C++ workflow + cpp_workflow = CppWorkflow(self.headnode, range_id) - if not node_py: - # In the first recursive state, just set the - # current DistRDF node as the head node - node_py = self.headnode - else: - # Execute the current operation using the output of the parent - # node (node_cpp) - RDFOperation = getattr(node_cpp, node_py.operation.name) - operation = node_py.operation + logger.debug("Generated C++ workflow is:\n{}".format(cpp_workflow)) - if operation.name == "Snapshot": - # Retrieve filename and append range boundaries - filename = operation.args[1].partition(".root")[0] - path_with_range = "{}_{}.root".format(filename, range_id) - # Create a partial snapshot on the current range - operation.args[1] = path_with_range - pyroot_node = RDFOperation(*operation.args, - **operation.kwargs) - - # The result is a pyroot object which is stored together with - # the pyrdf node. This binds the pyroot object lifetime to the - # pyrdf node, so both nodes will be kept alive as long as there - # is a valid reference poiting to the pyrdf node. - node_py.pyroot_node = pyroot_node - - # The new pyroot_node becomes the parent_node for the next - # recursive call - parent_node = pyroot_node - - if (node_py.operation.is_action() or - node_py.operation.is_instant_action()): - # Collect all action nodes in order to return them - # If it's a distributed snapshot return only path to - # the file with the partial snapshot - if operation.name == "Snapshot": - return_vals.append([path_with_range]) - else: - return_vals.append(pyroot_node) - - for n in node_py.children: - # Recurse through children and get their output - prev_vals = generate_computation_graph( - parent_node, range_id, node_py=n) - - # Attach the output of the children node - return_vals.extend(prev_vals) - - return return_vals - - return generate_computation_graph + # Compile and run the C++ workflow on the received RDF head node + return cpp_workflow.execute(ROOT.RDF.AsRNode(rdf_node)) + + def explore_graph(py_node, cpp_workflow, range_id, parent_idx): + """ + Recursively traverses the DistRDF graph nodes in DFS order and, + for each of them, adds a new node to the C++ workflow. + + Args: + py_node (Node): Object that contains the information to add the + corresponding node to the C++ workflow. + cpp_workflow (CppWorkflow): Object that encapsulates the creation + of the C++ workflow graph. + range_id (int): Id of the current range. Needed to assign a name to a + partial Snapshot output file. + parent_idx (int): Index of the parent node in the C++ workflow. + """ + node_idx = cpp_workflow.add_node(py_node.operation, range_id, parent_idx) + + for child_node in py_node.children: + explore_graph(child_node, cpp_workflow, range_id, node_idx) + + return run_computation_graph diff --git a/bindings/experimental/distrdf/python/DistRDF/CppWorkflow.py b/bindings/experimental/distrdf/python/DistRDF/CppWorkflow.py new file mode 100644 index 0000000000000..ec1184a46e774 --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/CppWorkflow.py @@ -0,0 +1,548 @@ +# @author Enric Tejedor +# @date 2021-07 + +################################################################################ +# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. # +# All rights reserved. # +# # +# For the licensing terms see $ROOTSYS/LICENSE. # +# For the list of contributors see $ROOTSYS/README/CREDITS. # +################################################################################ + +import os +from collections import namedtuple + +import ROOT +RunGraphs = ROOT.RDF.RunGraphs + +class CppWorkflow(object): + ''' + Class that encapsulates the generation of the code of an RDataFrame workflow + in C++, together with its compilation into a shared library and execution. + + This class is used by worker processes to execute in C++ the RDataFrame + graph they receive. This is done for the sake of performance, since running + the graph from Python necessarily relies on jitted code, which is less + optimized and thus slower than a shared library compiled with ACLiC. + + Attributes: + _cached_wfs (dict): uses as key the code of workflow functions that have + been already compiled and loaded by the current process, while the + value is the id of a given workflow function. Used to prevent + recompilation of already executed workflow functions. + + _FUNCTION_NAME (string): name of the function that encapsulates the + RDataFrame graph creation + + _FUNCTION_NAMESPACE (string): namespace of the function that + encapsulates the RDataFrame graph creation + + _graph_nodes (string): statements that add nodes to the graph. + + _includes (string): include statements needed by the workflow. + + _lambdas (string): lambda functions used by the workflow. + + _lambda_id (int): counter used to generate ids for each defined lambda + function. + + _node_id (int): counter used to generate ids for each graph node. + + _PyActionData (named tuple type): pair `(res_id,operation)`, where + `res_id` is the Python action result index in the vector of + workflow results and `operation` is the Operation object + corresponding to the action. + + _py_actions (list): list that contains _PyActionData objects. + + _res_ptr_id (int): counter used to generate ids for each result + generated by graph actions. + + _SnapshotData (named tuple type): pair `(res_id,filename)`, where + `res_id` is the Snapshot result index in the vector of workflow + results and `filename` is its modified output file name. + + _snapshots (list): list that contains _SnapshotData objects + + _wf_id (int): used to assign new ids to workflow functions + ''' + + _FUNCTION_NAME = '__RDF_WORKFLOW_FUNCTION__' + _FUNCTION_NAMESPACE = 'DistRDF_Internal' + + _cached_wfs = {} + _wf_id = 0 + + _SnapshotData = namedtuple('SnapshotData', ['res_id', 'filename']) + _PyActionData = namedtuple('PyActionData', ['res_id', 'operation']) + + def __init__(self, head_node, range_id): + ''' + Generates the C++ code of an RDF workflow that corresponds to the + received graph and data range. + + Args: + head_node (Node): head node of a graph that represents an RDF + workflow. + range_id (int): id of the data range to be processed by this + workflow. Needed to assign a name to a partial Snapshot output + file. + ''' + + self._includes = ''' +#include "ROOT/RDataFrame.hxx" +#include "ROOT/RResultHandle.hxx" +#include "ROOT/RDFHelpers.hxx" + +#include +#include +#include +''' + + self._lambdas = '' + self._lambda_id = 0 + + self._graph_nodes = '' + self._node_id = 1 # 0 is the head node we receive + + self._res_ptr_id = 0 + + self._snapshots = [] + + self._py_actions = [] + + # Generate the C++ workflow. + # Recurse over children nodes of received graph head node + head_node_id = 0 + for child_node in head_node.children: + self._explore_graph(child_node, range_id, head_node_id) + + def _explore_graph(self, node, range_id, parent_id): + """ + Recursively traverses the graph nodes in DFS order and, for each of + them, adds a new node to the C++ workflow. + + Args: + node (Node): object that contains the information to add the + corresponding node to the C++ workflow. + range_id (int): id of the current range. Needed to assign a name to a + partial Snapshot output file. + parent_id (int): id of the parent node in the C++ workflow. + """ + node_id = self.add_node(node.operation, range_id, parent_id) + + for child_node in node.children: + self._explore_graph(child_node, range_id, node_id) + + def add_include(self, header): + ''' + Adds a new include statement. + + Args: + header (str): header to be included. + ''' + + self._includes += '\n#include "{}"'.format(header) + + def add_lambda(self, lambda_code): + ''' + Adds a new lambda to be defined, which is needed by some operation + in the workflow. + + Args: + lambda_code (str): code of the lambda to be defined. + ''' + + self._lambdas += '\n auto rdf_lambda{n} = {code};' \ + .format(n=self._lambda_id, code=lambda_code) + self._lambda_id += 1 + + def add_node(self, operation, range_id, parent_id): + ''' + Adds the corresponding statement to add a new node in the RDataFrame + graph. What the statement returns depends on the type of the + operation: + - Transformation: the statement produces a new dataset node. + - Action: the statement returns an RResultPtr, which is added to a + vector of results to be returned at the end of the workflow + generation function. The type of the elements of the vector is + ROOT::RDF::RResultHandle to do type erasure since there could be + multiple booked actions with results of different types (e.g. integers, + TH1D, etc.). + + Args: + operation (Operation): object representing the operation to be added + to the graph. + + range_id (int): id of the current range. Needed to assign a name to + a partial Snapshot output file. + + parent_id (int): id of the parent node in this workflow. Used to + connect the new node to be added with its parent. + + Returns: + int: identifier of the node that was added to the graph + corresponding to its DFS order, if the operation is a + transformation, or None, if the operation is an action. + ''' + + # Operations that need special treatment + if operation.name == 'AsNumpy': + self._handle_asnumpy(operation, parent_id) + return None # nothing else to do + + if operation.name == 'Snapshot': + self._handle_snapshot(operation, range_id) # this modifies operation.args + + # Generate the code of the call that creates the new node + op_call = 'rdf{n}.{op}{templ_args}({args});' \ + .format(n=parent_id, op=operation.name, \ + templ_args=self._get_template_args(operation), \ + args=self._get_call_args(operation)) + + if operation.is_transformation(): + new_node_id = self._node_id + self._graph_nodes += '\n auto rdf{n} = {call}' \ + .format(n=new_node_id, call=op_call) + self._node_id += 1 + return new_node_id + + # Else it's an action or instant action + self._graph_nodes += '\n auto res_ptr{n} = {call}' \ + .format(n=self._res_ptr_id, call=op_call) + + # The result is stored in the vector of results to be returned + self._graph_nodes += '\n result_handles.emplace_back(res_ptr{});' \ + .format(self._res_ptr_id) + + # The result type is stored in the vector of result types to be + # returned + self._graph_nodes += \ + '\n auto c{0} = TClass::GetClass(typeid(res_ptr{0}));' \ + '\n if (c{0} == nullptr)' \ + '\n throw std::runtime_error(' \ + '\n "Cannot get type of result {0} of action {1} during "' \ + '\n "generation of RDF C++ workflow");' \ + '\n result_types.emplace_back(c{0}->GetName());' \ + .format(self._res_ptr_id, operation.name) + + self._res_ptr_id += 1 + + def _handle_snapshot(self, operation, range_id): + ''' + Does two extra settings needed for Snapshot nodes: + - Modifies the output file name to be of the form `filename_rangeid`, + since it is a partial snapshot for a given range. + - Stores the index of the returned vector in which the + result of this Snapshot is stored, together with the modified file + path. + + Args: + operation (Operation): object representing the operation to be added + to the graph. + + range_id (int): id of the current range. Needed to assign a name to + a partial Snapshot output file. + ''' + + # Modify file name + filename = operation.args[1].partition('.root')[0] + path_with_range = '{filename}_{rangeid}.root' \ + .format(filename=filename, rangeid=range_id) + operation.args[1] = path_with_range + + # Store Snapshot result index -> path + self._snapshots.append( + CppWorkflow._SnapshotData(self._res_ptr_id, path_with_range)) + + def _handle_asnumpy(self, operation, parent_id): + ''' + Since AsNumpy is a Python-only action, it can't be included in the + C++ workflow built by this class. Therefore, this function takes care + of saving the RDF node, generated in C++, on which an AsNumpy action + should be applied from Python. + + Args: + operation (Operation): object representing the AsNumpy operation + + parent_id (int): id of the parent node in this workflow. Used to + save that node in the vector of output nodes. + ''' + + # Store DFS-order index of the AsNumpy operation, together with the + # operation information, for later invocation from Python + self._py_actions.append(CppWorkflow._PyActionData(self._res_ptr_id, + operation)) + self._res_ptr_id += 1 + + # Save parent RDF node to run AsNumpy on it later from Python + self._graph_nodes += \ + '\n output_nodes.push_back(ROOT::RDF::AsRNode(rdf{}));' \ + .format(parent_id) + + # Add placeholders to the result lists + self._graph_nodes += '\n result_handles.emplace_back();' + self._graph_nodes += '\n result_types.emplace_back();' + + def _get_template_args(self, operation): + ''' + Gets the template arguments with which to generate the call to a given + operation. + + Args: + operation (Operation): object representing the operation whose + template arguments need to be returned. + + Returns: + string: template arguments for this operation. + ''' + + # TODO: generate templated operations when possible, e.g. Max + + return '' + + def _get_call_args(self, operation): + ''' + Gets the arguments with which to generate the call to a given operation. + + Args: + operation (Operation): object representing the operation whose + call arguments need to be returned. + + Returns: + string: call arguments for this operation. + ''' + + # TODO + # - Do a more thorough type conversion + # - Use RDF helper functions to convert jitted strings to lambdas + + args = "" + + # Argument type conversion + for narg, arg in enumerate(operation.args): + if (narg > 0): + args += ', ' + + if isinstance(arg, str): + args += '"{}"'.format(arg) + elif isinstance(arg, tuple): + args += '{' + for nelem, elem in enumerate(arg): + if nelem > 0: + args += ',' + if isinstance(elem, str): + args += '"{}"'.format(elem) + else: + args += '{}'.format(elem) + args += '}' + elif isinstance(arg, ROOT.RDF.RSnapshotOptions): + ROOT.Warning('DistRDF', + 'Processing of RSnapshotOptions not implemented') + + # Make Snapshot lazy + # TODO: Do a proper processing of the args (user might have specified + # her own options object) + # Remove warning above when this is implemented + if operation.name == 'Snapshot': + args += ', "", lazy_options' + + return args + + def execute(self, rdf): + ''' + Compiles the workflow generation code and executes it. + + Args: + rdf (ROOT::RDF::RNode): object that represents the dataset on + which to execute the workflow. + + Returns: + tuple: the first element is the list of results of the actions in + the C++ workflow, the second element is the list of result types + corresponding to those actions. + ''' + + wf_id = self._compile() + return self._run_function(rdf, wf_id) + + def _compile(self): + ''' + Generates the workflow code C++ file and compiles it with ACLiC + into a shared library. The library is also loaded as part of the + `TSystem::CompileMacro` call. + + The name of the generated C++ file contains both a hash of its + code and the ID of the process that created it. This is done to + prevent clashes between multiple (non-sandboxed) worker processes + that try to write to the same file concurrently. + + A class-level cache keeps track of the workflows that have been already + compiled to prevent unncessary recompilation (e.g. when a worker + process runs multiple times the same workflow). + + Returns: + int: the id of the workflow function to be executed. Such id is + appended to CppWorkflow._FUNCTION_NAME to prevent name clashes + (a worker process might compile and load multiple workflow + functions). + ''' + + # TODO: Make this function thread-safe? To support Dask threaded + # workers + + code = self._get_code() + this_wf_id = CppWorkflow._cached_wfs.get(code) + if this_wf_id is not None: + # We already compiled and loaded a workflow function with this + # code. Return the id of that function + return this_wf_id + + # We are trying to run this workflow for the first time in this + # process. First dump the code in a file with the right function name + this_wf_id = CppWorkflow._wf_id + cpp_file_name = 'rdfworkflow_{wf_id}_{pid}.cxx' \ + .format(wf_id=this_wf_id, pid=os.getpid()) + final_code = code.replace(CppWorkflow._FUNCTION_NAME, + CppWorkflow._FUNCTION_NAME + str(this_wf_id), + 1) + with open(cpp_file_name, 'w') as f: + f.write(final_code) + + # Now compile and load the code + if not ROOT.gSystem.CompileMacro(cpp_file_name, 'O'): + raise RuntimeError( + 'Error compiling the RDataFrame workflow file: {}' \ + .format(cpp_file_name)) + + # Let the cache know there is a new workflow + CppWorkflow._cached_wfs[code] = this_wf_id + CppWorkflow._wf_id += 1 + + return this_wf_id + + def _run_function(self, rdf, wf_id): + ''' + Runs the workflow generation function. + + Args: + rdf (ROOT::RDF::RNode): object that represents the dataset on + which to execute the workflow. + wf_id (int): identifier of the workflow function to be executed. + + Returns: + tuple: the first element is the list of results of the actions in + the C++ workflow, the second element is the list of result types + corresponding to those actions. + ''' + + ns = getattr(ROOT, CppWorkflow._FUNCTION_NAMESPACE) + func = getattr(ns, CppWorkflow._FUNCTION_NAME + str(wf_id)) + + # Run the workflow generator function + vectors = func(rdf) # need to keep the tuple alive + v_results, v_res_types, v_nodes = vectors + + # Convert the vector of results into a list so that we can mix + # different types in it. + # We copy the results since the life of the original ones is tied to + # that of the vector + results = [ ROOT.RDF.RResultHandle(res) for res in v_results ] + + # Strip out the ROOT::RDF::RResultPtr<> part of the type + def get_result_type(s): + if s.empty(): + # Python-only actions have an empty return type in C++ + return '' + + s = str(s) + pos = s.find('<') + if pos == -1: + raise RuntimeError( + 'Error parsing the result types of RDataFrame workflow') + return s[pos+1:-1].strip() + + res_types = [ get_result_type(elem) for elem in v_res_types ] + + # Add Python-only actions on their corresponding nodes + for (i, operation), n in zip(self._py_actions, v_nodes): + operation.kwargs['lazy'] = True # make it lazy + results[i] = getattr(n, operation.name)(*operation.args, **operation.kwargs) + + if v_results: + # We trigger the event loop here, so make sure we release the GIL + old_rg = RunGraphs.__release_gil__ + RunGraphs.__release_gil__ = True + RunGraphs(v_results) + RunGraphs.__release_gil__ = old_rg + + # Replace the RResultHandle of each Snapshot by its modified output + # path, since the latter is what we actually need in the reducer + for i, path in self._snapshots: + results[i] = [path] + res_types[i] = None # placeholder + + # Replace the future-like result of every Python-only action (e.g. + # AsNumpyResult) by its actual value + for i, operation in self._py_actions: + results[i] = results[i].GetValue() + + return results, res_types + + def __repr__(self): + ''' + Generates a string representation for this C++ workflow. + + Returns: + string: code of this C++ workflow. + ''' + + return self._get_code() + + def _get_code(self): + ''' + Composes the workflow generation code from the different attributes + of this class. The resulting code contains a function that will be + called to generate the RDataFrame graph. Such function returns a tuple + of three elements: + 1. A vector of results of the graph actions. + 2. A vector with the result types of those actions. + 3. A vector of RDF nodes that will be used in Python to invoke + Python-only actions on them (e.g. `AsNumpy`). + ''' + + code = ''' +{includes} + +namespace {namespace} {{ + +using CppWorkflowResult = std::tuple, + std::vector, + std::vector>; + +CppWorkflowResult {func_name}(ROOT::RDF::RNode &rdf0) +{{ + std::vector result_handles; + std::vector result_types; + std::vector output_nodes; + + // To make Snapshots lazy + ROOT::RDF::RSnapshotOptions lazy_options; + lazy_options.fLazy = true; + +{lambdas} + +{nodes} + + return {{ std::move(result_handles), std::move(result_types), std::move(output_nodes) }}; +}} + +}} +'''.format(func_name=CppWorkflow._FUNCTION_NAME, + namespace=CppWorkflow._FUNCTION_NAMESPACE, + includes=self._includes, + lambdas=self._lambdas, + nodes=self._graph_nodes) + + return code + diff --git a/bindings/experimental/distrdf/python/DistRDF/Operation.py b/bindings/experimental/distrdf/python/DistRDF/Operation.py index 791528793774f..9bd7bff65796a 100644 --- a/bindings/experimental/distrdf/python/DistRDF/Operation.py +++ b/bindings/experimental/distrdf/python/DistRDF/Operation.py @@ -62,6 +62,7 @@ def _classify_operation(self, name): operations_dict = { "Define": Operation.TRANSFORMATION, + "DefinePerSample": Operation.TRANSFORMATION, "Filter": Operation.TRANSFORMATION, "Range": Operation.TRANSFORMATION, "Aggregate": Operation.ACTION, diff --git a/bindings/experimental/distrdf/python/DistRDF/PythonMergeables.py b/bindings/experimental/distrdf/python/DistRDF/PythonMergeables.py new file mode 100644 index 0000000000000..ee1394d909026 --- /dev/null +++ b/bindings/experimental/distrdf/python/DistRDF/PythonMergeables.py @@ -0,0 +1,44 @@ + +import ROOT + + +class SnapshotResult(object): + """ + Encapsulate information coming from a Snapshot operation and know how to + merge it with other objects of this type. + """ + + def __init__(self, treename, filenames): + self.treename = treename + self.filenames = filenames + + def Merge(self, other): + """ + When calling Snapshot on a distributed worker, a list with the path to + the snapshotted file on the worker is stored. This function extends the + list of the current object with the elements from the list of the other + object. + """ + self.filenames.extend(other.filenames) + + def GetValue(self, backend): + """ + With local RDataFrame, Snapshot returns another RDataFrame object that + can be used to continue the application. The equivalent in the + distributed scenario is to create a distributed RDataFrame. + + This is done by constructing a TChain with the name and the list of + paths stored in this object. The chain is then passed to the + `make_dataframe` function that changes depending on the backend. + + For example, if the original RDataFrame that triggered the distributed + computation was created via a Spark backend, then this function will + return another distributed RDataFrame build from a Spark backend + instance. And so on for all other DistRDF backends. + """ + snapshot_chain = ROOT.TChain(self.treename) + # Add partial snapshot files to the chain + for filename in self.filenames: + snapshot_chain.Add(filename) + # Create a new rdf with the chain and return that to user + return backend.make_dataframe(snapshot_chain) diff --git a/bindings/experimental/distrdf/python/DistRDF/__init__.py b/bindings/experimental/distrdf/python/DistRDF/__init__.py index f9f405fc3094f..d9d7357c12d75 100644 --- a/bindings/experimental/distrdf/python/DistRDF/__init__.py +++ b/bindings/experimental/distrdf/python/DistRDF/__init__.py @@ -85,4 +85,7 @@ def create_distributed_module(parentmodule): distributed.initialize = initialize distributed.create_logger = create_logger + # Set non-optimized default mode + distributed.optimized = False + return distributed diff --git a/bindings/experimental/distrdf/test/backend/test_spark.py b/bindings/experimental/distrdf/test/backend/test_spark.py index d8de4233016f5..e177ba8f9e5b0 100644 --- a/bindings/experimental/distrdf/test/backend/test_spark.py +++ b/bindings/experimental/distrdf/test/backend/test_spark.py @@ -1,6 +1,7 @@ import os import sys import unittest +import warnings import DistRDF import pyspark @@ -18,24 +19,33 @@ class SparkBackendInitTest(unittest.TestCase): @classmethod def setUpClass(cls): """ - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Set up test environment for this class. Currently this includes: - Needed to avoid mismatch between python versions on driver and on - the fake executor on the same machine. + - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Needed to avoid mismatch between python versions on driver and on the + fake executor on the same machine. + - Ignore `ResourceWarning: unclosed socket` warning triggered by Spark. + this is ignored by default in any application, but Python's unittest + library overrides the default warning filters thus exposing this + warning """ os.environ["PYSPARK_PYTHON"] = sys.executable - def tearDown(self): - """Stop any created SparkContext""" - pyspark.SparkContext.getOrCreate().stop() + if sys.version_info.major >= 3: + warnings.simplefilter("ignore", ResourceWarning) @classmethod def tearDownClass(cls): - """ - Stop the SparkContext and reset environment variable. - """ + """Reset test environment.""" os.environ["PYSPARK_PYTHON"] = "" + if sys.version_info.major >= 3: + warnings.simplefilter("default", ResourceWarning) + + def tearDown(self): + """Stop any created SparkContext""" + pyspark.SparkContext.getOrCreate().stop() + def test_set_spark_context_default(self): """ Check that a `SparkContext` object is created with default options for @@ -60,17 +70,19 @@ def test_set_spark_context_with_conf(self): appname = backend.sc.getConf().get("spark.app.name") self.assertEqual(appname, "my-pyspark-app1") - def test_optimize_npartitions_with_num_executors(self): + def test_optimize_npartitions(self): """ - Check that the number of partitions is correctly set to number of - executors in the SparkConf dictionary. + The optimize_npartitions function returns the value of the + `defaultParallelism` attribute of the `SparkContext`. This should be + equal to the number of available cores in case of a context created on + a single machine. """ - conf = {"spark.executor.instances": 10} - sconf = pyspark.SparkConf().setAll(conf.items()) + ncores = 4 + sconf = pyspark.SparkConf().setMaster(f"local[{ncores}]") sc = pyspark.SparkContext(conf=sconf) backend = Backend.SparkBackend(sparkcontext=sc) - self.assertEqual(backend.optimize_npartitions(), 10) + self.assertEqual(backend.optimize_npartitions(), ncores) class OperationSupportTest(unittest.TestCase): @@ -82,24 +94,33 @@ class OperationSupportTest(unittest.TestCase): @classmethod def setUpClass(cls): """ - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Set up test environment for this class. Currently this includes: - Needed to avoid mismatch between python versions on driver and on - the fake executor on the same machine. + - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Needed to avoid mismatch between python versions on driver and on the + fake executor on the same machine. + - Ignore `ResourceWarning: unclosed socket` warning triggered by Spark. + this is ignored by default in any application, but Python's unittest + library overrides the default warning filters thus exposing this + warning """ os.environ["PYSPARK_PYTHON"] = sys.executable - def tearDown(self): - """Stop any created SparkContext""" - pyspark.SparkContext.getOrCreate().stop() + if sys.version_info.major >= 3: + warnings.simplefilter("ignore", ResourceWarning) @classmethod def tearDownClass(cls): - """ - Stop the SparkContext and reset environment variable. - """ + """Reset test environment.""" os.environ["PYSPARK_PYTHON"] = "" + if sys.version_info.major >= 3: + warnings.simplefilter("default", ResourceWarning) + + def tearDown(self): + """Stop any created SparkContext""" + pyspark.SparkContext.getOrCreate().stop() + def test_action(self): """Check that action nodes are classified accurately.""" backend = Backend.SparkBackend() @@ -144,24 +165,33 @@ class InitializationTest(unittest.TestCase): @classmethod def setUpClass(cls): """ - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Set up test environment for this class. Currently this includes: - Needed to avoid mismatch between python versions on driver and on - the fake executor on the same machine. + - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Needed to avoid mismatch between python versions on driver and on the + fake executor on the same machine. + - Ignore `ResourceWarning: unclosed socket` warning triggered by Spark. + this is ignored by default in any application, but Python's unittest + library overrides the default warning filters thus exposing this + warning """ os.environ["PYSPARK_PYTHON"] = sys.executable - def tearDown(self): - """Stop any created SparkContext""" - pyspark.SparkContext.getOrCreate().stop() + if sys.version_info.major >= 3: + warnings.simplefilter("ignore", ResourceWarning) @classmethod def tearDownClass(cls): - """ - Stop the SparkContext and reset environment variable. - """ + """Reset test environment.""" os.environ["PYSPARK_PYTHON"] = "" + if sys.version_info.major >= 3: + warnings.simplefilter("default", ResourceWarning) + + def tearDown(self): + """Stop any created SparkContext""" + pyspark.SparkContext.getOrCreate().stop() + def test_initialization(self): """ Check that the user initialization method is assigned to the current @@ -219,24 +249,33 @@ class EmptyTreeErrorTest(unittest.TestCase): @classmethod def setUpClass(cls): """ - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Set up test environment for this class. Currently this includes: - Needed to avoid mismatch between python versions on driver and on - the fake executor on the same machine. + - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Needed to avoid mismatch between python versions on driver and on the + fake executor on the same machine. + - Ignore `ResourceWarning: unclosed socket` warning triggered by Spark. + this is ignored by default in any application, but Python's unittest + library overrides the default warning filters thus exposing this + warning """ os.environ["PYSPARK_PYTHON"] = sys.executable - def tearDown(self): - """Stop any created SparkContext""" - pyspark.SparkContext.getOrCreate().stop() + if sys.version_info.major >= 3: + warnings.simplefilter("ignore", ResourceWarning) @classmethod def tearDownClass(cls): - """ - Stop the SparkContext and reset environment variable. - """ + """Reset test environment.""" os.environ["PYSPARK_PYTHON"] = "" + if sys.version_info.major >= 3: + warnings.simplefilter("default", ResourceWarning) + + def tearDown(self): + """Stop any created SparkContext""" + pyspark.SparkContext.getOrCreate().stop() + def test_histo_from_empty_root_file(self): """ Check that when performing operations with the distributed backend on @@ -259,24 +298,33 @@ class ChangeAttributeTest(unittest.TestCase): @classmethod def setUpClass(cls): """ - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Set up test environment for this class. Currently this includes: - Needed to avoid mismatch between python versions on driver and on - the fake executor on the same machine. + - Synchronize PYSPARK_PYTHON variable to the current Python executable. + Needed to avoid mismatch between python versions on driver and on the + fake executor on the same machine. + - Ignore `ResourceWarning: unclosed socket` warning triggered by Spark. + this is ignored by default in any application, but Python's unittest + library overrides the default warning filters thus exposing this + warning """ os.environ["PYSPARK_PYTHON"] = sys.executable - def tearDown(self): - """Stop any created SparkContext""" - pyspark.SparkContext.getOrCreate().stop() + if sys.version_info.major >= 3: + warnings.simplefilter("ignore", ResourceWarning) @classmethod def tearDownClass(cls): - """ - Stop the SparkContext and reset environment variable. - """ + """Reset test environment.""" os.environ["PYSPARK_PYTHON"] = "" + if sys.version_info.major >= 3: + warnings.simplefilter("default", ResourceWarning) + + def tearDown(self): + """Stop any created SparkContext""" + pyspark.SparkContext.getOrCreate().stop() + def test_change_attribute_when_npartitions_greater_than_clusters(self): """ Check that the `npartitions class attribute is changed when it is @@ -294,22 +342,6 @@ def test_change_attribute_when_npartitions_greater_than_clusters(self): self.assertEqual(nentries, 10) self.assertEqual(df._headnode.npartitions, 1) - def test_optimize_npartitions_with_spark_config_options(self): - """ - Check that relevant spark configuration options optimize the number of - partitions. - """ - - conf = {"spark.executor.cores": 4, "spark.executor.instances": 4} - sconf = pyspark.SparkConf().setAll(conf.items()) - scontext = pyspark.SparkContext(conf=sconf) - - df = Spark.RDataFrame(100, sparkcontext=scontext) - - # The number of partitions was optimized to be equal to - # spark.executor.cores * spark.executor.instances - self.assertEqual(df._headnode.npartitions, 16) - def test_user_supplied_npartitions_have_precedence(self): """ Check that even if spark configuration options could optimize the number diff --git a/bindings/experimental/distrdf/test/test_callable_generator.py b/bindings/experimental/distrdf/test/test_callable_generator.py index ca3689939de63..66010fda3bd5c 100644 --- a/bindings/experimental/distrdf/test/test_callable_generator.py +++ b/bindings/experimental/distrdf/test/test_callable_generator.py @@ -78,15 +78,15 @@ def test_mapper_from_graph(self): # Generate and execute the mapper generator = ComputationGraphGenerator.ComputationGraphGenerator( node.proxied_node) - mapper_func = generator.get_callable() - values = mapper_func(t, 0) + mapper_func = generator.generate_computation_graph + triggerables = mapper_func(t, 0) nodes = generator.get_action_nodes() reqd_order = [1, 3, 2, 2, 3, 2] self.assertEqual(t.ord_list, reqd_order) self.assertListEqual(nodes, [n5.proxied_node, n4.proxied_node]) - self.assertListEqual(values, [t, t]) + self.assertListEqual(triggerables, [t, t]) def test_mapper_with_pruning(self): """ @@ -115,12 +115,14 @@ def test_mapper_with_pruning(self): # Generate and execute the mapper generator = ComputationGraphGenerator.ComputationGraphGenerator( node.proxied_node) - mapper_func = generator.get_callable() - values = mapper_func(t, 0) + # Prune first + generator.headnode.graph_prune() + mapper_func = generator.generate_computation_graph + triggerables = mapper_func(t, 0) nodes = generator.get_action_nodes() reqd_order = [1, 2, 2, 2, 3, 2] self.assertEqual(t.ord_list, reqd_order) self.assertListEqual(nodes, [n4.proxied_node]) - self.assertListEqual(values, [t]) + self.assertListEqual(triggerables, [t]) diff --git a/bindings/jupyroot/python/JupyROOT/__init__.py b/bindings/jupyroot/python/JupyROOT/__init__.py index 0fb8e13195f5a..30a287a4b862e 100644 --- a/bindings/jupyroot/python/JupyROOT/__init__.py +++ b/bindings/jupyroot/python/JupyROOT/__init__.py @@ -11,7 +11,10 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. # ################################################################################ -from JupyROOT.helpers import cppcompleter, utils +import sys +from JupyROOT.helpers import utils +if not 'win32' in sys.platform: + from JupyROOT.helpers import cppcompleter # Check if we are in the IPython shell try: @@ -21,6 +24,7 @@ _is_ipython = hasattr(builtins, '__IPYTHON__') if _is_ipython: - from IPython import get_ipython - cppcompleter.load_ipython_extension(get_ipython()) + if not 'win32' in sys.platform: + from IPython import get_ipython + cppcompleter.load_ipython_extension(get_ipython()) utils.iPythonize() diff --git a/bindings/jupyroot/python/JupyROOT/helpers/utils.py b/bindings/jupyroot/python/JupyROOT/helpers/utils.py index 3c404099293bb..babc2ce7242dc 100644 --- a/bindings/jupyroot/python/JupyROOT/helpers/utils.py +++ b/bindings/jupyroot/python/JupyROOT/helpers/utils.py @@ -19,7 +19,6 @@ import sys import select import tempfile -import pty import itertools import re import fnmatch @@ -561,10 +560,12 @@ def _jsDisplay(self): return 0 def _getPngImage(self): - ofile = tempfile.NamedTemporaryFile(suffix=".png") + ofile = tempfile.NamedTemporaryFile(suffix=".png", delete=False) with _setIgnoreLevel(ROOT.kError): self.drawableObject.SaveAs(ofile.name) img = IPython.display.Image(filename=ofile.name, format='png', embed=True) + ofile.close() + os.unlink(ofile.name) return img def _pngDisplay(self): diff --git a/bindings/jupyroot/src/IOHandler.cxx b/bindings/jupyroot/src/IOHandler.cxx index 6a2cc2766f9aa..40f29f8ad25b7 100644 --- a/bindings/jupyroot/src/IOHandler.cxx +++ b/bindings/jupyroot/src/IOHandler.cxx @@ -71,15 +71,25 @@ JupyROOTExecutorHandler::JupyROOTExecutorHandler() {} static void PollImpl(FILE *stdStream, int *pipeHandle, std::string &pipeContent) { + fflush(stdStream); +#ifdef _MSC_VER + char buffer[60000] = ""; + struct _stat st; + _fstat(pipeHandle[0], &st); + if (st.st_size) { + _read(pipeHandle[0], buffer, 60000); + pipeContent += buffer; + } +#else int buf_read; char ch; - fflush(stdStream); while (true) { buf_read = read(pipeHandle[0], &ch, 1); if (buf_read == 1) { pipeContent += ch; } else break; } +#endif } void JupyROOTExecutorHandler::Poll() @@ -94,10 +104,7 @@ static void InitCaptureImpl(int &savedStdStream, int *pipeHandle, int FILENO) if (pipe(pipeHandle) != 0) { return; } -#ifdef _MSC_VER // Visual Studio - unsigned long mode = 1; - ioctlsocket(pipeHandle[0], FIONBIO, &mode); -#else +#ifndef _MSC_VER long flags_stdout = fcntl(pipeHandle[0], F_GETFL); if (flags_stdout == -1) return; flags_stdout |= O_NONBLOCK; diff --git a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt index 2144d97d580bb..d41b649883f51 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt +++ b/bindings/pyroot/cppyy/CPyCppyy/CMakeLists.txt @@ -108,7 +108,7 @@ foreach(val RANGE ${how_many_pythons}) ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries) if (NOT MSVC AND NOT CMAKE_INSTALL_LIBDIR STREQUAL CMAKE_INSTALL_PYTHONDIR) # add a symlink to ${libname} in CMAKE_INSTALL_PYTHONDIR - set(LIB_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${libname}${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(LIB_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${libname}.so) install(CODE "file(CREATE_LINK ${PYTHONDIR_TO_LIBDIR}/${LIB_FILE_NAME} \$ENV{DESTDIR}${CMAKE_INSTALL_FULL_PYTHONDIR}/${LIB_FILE_NAME} SYMBOLIC)") endif() diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx index c05ec97bdedf1..685ad3dc60937 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPPMethod.cxx @@ -409,7 +409,7 @@ int CPyCppyy::CPPMethod::GetPriority() const std::string& clean_name = TypeManip::clean_type(aname, false); Cppyy::TCppScope_t scope = Cppyy::GetScope(clean_name); if (scope) - priority += (int)Cppyy::GetNumBases(scope); + priority += static_cast(Cppyy::GetNumBasesLongestBranch(scope)); if (Cppyy::IsEnum(clean_name)) priority -= 100; diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx b/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx index 659111ceab553..411a57a862f5b 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx +++ b/bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx @@ -431,15 +431,14 @@ static PyObject* addressof(PyObject* /* dummy */, PyObject* args, PyObject* kwds // error message if not already set if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, "aap"); - /*if (PyTuple_CheckExact(args) && PyTuple_GET_SIZE(args)) { + if (PyTuple_CheckExact(args) && PyTuple_GET_SIZE(args)) { PyObject* str = PyObject_Str(PyTuple_GET_ITEM(args, 0)); if (str && CPyCppyy_PyText_Check(str)) PyErr_Format(PyExc_TypeError, "unknown object %s", CPyCppyy_PyText_AsString(str)); else PyErr_Format(PyExc_TypeError, "unknown object at %p", (void*)PyTuple_GET_ITEM(args, 0)); Py_XDECREF(str); - } */ + } } return nullptr; } diff --git a/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h b/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h index ea4d222050d49..f85a922dc5959 100644 --- a/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h +++ b/bindings/pyroot/cppyy/CPyCppyy/src/Cppyy.h @@ -137,6 +137,8 @@ namespace Cppyy { CPPYY_IMPORT TCppIndex_t GetNumBases(TCppType_t type); CPPYY_IMPORT + TCppIndex_t GetNumBasesLongestBranch(TCppType_t type); + CPPYY_IMPORT std::string GetBaseName(TCppType_t type, TCppIndex_t ibase); CPPYY_IMPORT bool IsSubtype(TCppType_t derived, TCppType_t base); diff --git a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt index dda8b4848973c..da044999530c7 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt +++ b/bindings/pyroot/cppyy/cppyy-backend/CMakeLists.txt @@ -49,7 +49,7 @@ foreach(val RANGE ${how_many_pythons}) ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries) if (NOT MSVC AND NOT CMAKE_INSTALL_LIBDIR STREQUAL CMAKE_INSTALL_PYTHONDIR) # add a symlink to ${libname} in CMAKE_INSTALL_PYTHONDIR - set(LIB_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${libname}${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(LIB_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${libname}.so) install(CODE "file(CREATE_LINK ${PYTHONDIR_TO_LIBDIR}/${LIB_FILE_NAME} \$ENV{DESTDIR}${CMAKE_INSTALL_FULL_PYTHONDIR}/${LIB_FILE_NAME} SYMBOLIC)") endif() diff --git a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/capi.h b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/capi.h index 2ea998a33d2cb..f3bef757ccf6e 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/capi.h +++ b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/capi.h @@ -124,6 +124,8 @@ extern "C" { RPY_EXPORTED int cppyy_num_bases(cppyy_type_t type); RPY_EXPORTED + int cppyy_num_bases_longest_branch(cppyy_type_t type); + RPY_EXPORTED char* cppyy_base_name(cppyy_type_t type, int base_index); RPY_EXPORTED int cppyy_is_subtype(cppyy_type_t derived, cppyy_type_t base); diff --git a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx index 4883e40bd0404..edb5533623137 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx +++ b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/clingwrapper.cxx @@ -1219,6 +1219,83 @@ Cppyy::TCppIndex_t Cppyy::GetNumBases(TCppType_t klass) return (TCppIndex_t)0; } +//////////////////////////////////////////////////////////////////////////////// +/// \fn Cppyy::TCppIndex_t GetLongestInheritancePath(TClass *klass) +/// \brief Retrieve number of base classes in the longest branch of the +/// inheritance tree of the input class. +/// \param[in] klass The class to start the retrieval process from. +/// +/// This is a helper function for Cppyy::GetNumBasesLongestBranch. +/// Given an inheritance tree, the function assigns weight 1 to each class that +/// has at least one base. Starting from the input class, the function is +/// called recursively on all the bases. For each base the return value is one +/// (the weight of the base itself) plus the maximum value retrieved for their +/// bases in turn. For example, given the following inheritance tree: +/// +/// ~~~{.cpp} +/// class A {}; class B: public A {}; +/// class X {}; class Y: public X {}; class Z: public Y {}; +/// class C: public B, Z {}; +/// ~~~ +/// +/// calling this function on an instance of `C` will return 3, the steps +/// required to go from C to X. +Cppyy::TCppIndex_t GetLongestInheritancePath(TClass *klass) +{ + + auto directbases = klass->GetListOfBases(); + if (!directbases) { + // This is a leaf with no bases + return 0; + } + auto ndirectbases = directbases->GetSize(); + if (ndirectbases == 0) { + // This is a leaf with no bases + return 0; + } else { + // If there is at least one direct base + std::vector nbases_branches; + nbases_branches.reserve(ndirectbases); + + // Traverse all direct bases of the current class and call the function + // recursively + for (auto baseclass : TRangeDynCast(directbases)) { + if (!baseclass) + continue; + if (auto baseclass_tclass = baseclass->GetClassPointer()) { + nbases_branches.emplace_back(GetLongestInheritancePath(baseclass_tclass)); + } + } + + // Get longest path among the direct bases of the current class + auto longestbranch = std::max_element(std::begin(nbases_branches), std::end(nbases_branches)); + + // Add 1 to include the current class in the count + return 1 + *longestbranch; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// \fn Cppyy::TCppIndex_t Cppyy::GetNumBasesLongest(TCppType_t klass) +/// \brief Retrieve number of base classes in the longest branch of the +/// inheritance tree. +/// \param[in] klass The class to start the retrieval process from. +/// +/// The function converts the input class to a `TClass *` and calls +/// GetLongestInheritancePath. +Cppyy::TCppIndex_t Cppyy::GetNumBasesLongestBranch(TCppType_t klass) +{ + + const auto &cr = type_from_handle(klass); + + if (auto klass_tclass = cr.GetClass()) { + return GetLongestInheritancePath(klass_tclass); + } + + // In any other case, return zero + return 0; +} + std::string Cppyy::GetBaseName(TCppType_t klass, TCppIndex_t ibase) { TClassRef& cr = type_from_handle(klass); @@ -2383,6 +2460,10 @@ int cppyy_num_bases(cppyy_type_t type) { return (int)Cppyy::GetNumBases(type); } +int cppyy_num_bases_longest_branch(cppyy_type_t type) { + return (int)Cppyy::GetNumBasesLongestBranch(type); +} + char* cppyy_base_name(cppyy_type_t type, int base_index) { return cppstring_to_cstring(Cppyy::GetBaseName (type, base_index)); } diff --git a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h index e9fd3b26ab81e..7757c19d1a010 100644 --- a/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h +++ b/bindings/pyroot/cppyy/cppyy-backend/clingwrapper/src/cpp_cppyy.h @@ -134,6 +134,8 @@ namespace Cppyy { RPY_EXPORTED TCppIndex_t GetNumBases(TCppType_t type); RPY_EXPORTED + TCppIndex_t GetNumBasesLongestBranch(TCppType_t type); + RPY_EXPORTED std::string GetBaseName(TCppType_t type, TCppIndex_t ibase); RPY_EXPORTED bool IsSubtype(TCppType_t derived, TCppType_t base); diff --git a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py index a403f31b53512..dfdb58d90bf0a 100644 --- a/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py +++ b/bindings/pyroot/cppyy/cppyy/python/cppyy/__init__.py @@ -69,7 +69,7 @@ # tell cppyy that libcppyy_backend is versioned def _check_py_version(lib_name, cbl_var): import re - if re.match('^libcppyy_backend\d_\d$', lib_name): + if re.match('^libcppyy_backend\d+_\d+$', lib_name): # library name already has version if lib_name.endswith(py_version_str): return '' diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx b/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx index cf79b7d6ae0c9..c1d47b4016504 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.cxx @@ -1,5 +1,3 @@ -#if __cplusplus >= 201103L - #include "cpp11features.h" @@ -37,5 +35,3 @@ void implicit_converion_move(TestMoving2&&) { // for std::function testing std::function FNCreateTestStructFunc() { return [](const FNTestStruct& t) { return t.t; }; } std::function FunctionNS::FNCreateTestStructFunc() { return [](const FNTestStruct& t) { return t.t; }; } - -#endif // c++11 and later diff --git a/bindings/pyroot/cppyy/cppyy/test/cpp11features.h b/bindings/pyroot/cppyy/cppyy/test/cpp11features.h index e2609f8bacb98..f0eba340766b0 100644 --- a/bindings/pyroot/cppyy/cppyy/test/cpp11features.h +++ b/bindings/pyroot/cppyy/cppyy/test/cpp11features.h @@ -1,5 +1,3 @@ -#if __cplusplus >= 201103L - #include #include #include @@ -114,5 +112,3 @@ namespace std { size_t operator()(const StructWithHash&) const { return 17; } }; } // namespace std - -#endif // c++11 and later diff --git a/bindings/pyroot/pythonizations/python/ROOT/_facade.py b/bindings/pyroot/pythonizations/python/ROOT/_facade.py index 3d1c867fdf8fa..81759b0ca9d8a 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/_facade.py +++ b/bindings/pyroot/pythonizations/python/ROOT/_facade.py @@ -330,8 +330,9 @@ def RDF(self): from libROOTPythonizations import MakeNumpyDataFrame ns.MakeNumpyDataFrame = MakeNumpyDataFrame - # Inject Experimental.Distributed package into namespace RDF - ns.Experimental = _create_rdf_experimental_distributed_module(ns) + if sys.version_info >= (3, 7): + # Inject Experimental.Distributed package into namespace RDF + ns.Experimental = _create_rdf_experimental_distributed_module(ns) except: raise Exception('Failed to pythonize the namespace RDF') del type(self).RDF diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_rdataframe.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_rdataframe.py index 61d8ef9eaf303..89a3d84bb79e4 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_rdataframe.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_rdataframe.py @@ -25,7 +25,7 @@ def __get__(self, instance, owner): return partial(self.func, instance, *(self.args or ()), **(self.keywords or {})) -def RDataFrameAsNumpy(df, columns=None, exclude=None): +def RDataFrameAsNumpy(df, columns=None, exclude=None, lazy=False): """Read-out the RDataFrame as a collection of numpy arrays. The values of the dataframe are read out as numpy array of the respective type @@ -45,9 +45,12 @@ def RDataFrameAsNumpy(df, columns=None, exclude=None): Parameters: columns: If None return all branches as columns, otherwise specify names in iterable. exclude: Exclude branches from selection. + lazy: Determines whether this action is instant (False, default) or lazy (True). Returns: - dict: Dict with column names as keys and 1D numpy arrays with content as values + dict or AsNumpyResult: if instant (default), dict with column names as keys and + 1D numpy arrays with content as values; if lazy, AsNumpyResult containing + the result pointers obtained from the Take actions. """ # Sanitize input arguments if isinstance(columns, str): @@ -55,10 +58,9 @@ def RDataFrameAsNumpy(df, columns=None, exclude=None): if isinstance(exclude, str): raise TypeError("The exclude argument requires a list of strings") - # Import numpy and numpy.array derived class lazily + # Early check for numpy try: import numpy - from ROOT.pythonization._rdf_utils import ndarray except: raise ImportError("Failed to import numpy during call of RDataFrame.AsNumpy.") @@ -77,20 +79,117 @@ def RDataFrameAsNumpy(df, columns=None, exclude=None): column_type = df.GetColumnType(column) result_ptrs[column] = df.Take[column_type](column) - # Convert the C++ vectors to numpy arrays - py_arrays = {} - for column in columns: - cpp_reference = result_ptrs[column].GetValue() - if hasattr(cpp_reference, "__array_interface__"): - tmp = numpy.asarray(cpp_reference) # This adopts the memory of the C++ object. - py_arrays[column] = ndarray(tmp, result_ptrs[column]) - else: - tmp = numpy.empty(len(cpp_reference), dtype=numpy.object) - for i, x in enumerate(cpp_reference): - tmp[i] = x # This creates only the wrapping of the objects and does not copy. - py_arrays[column] = ndarray(tmp, result_ptrs[column]) + result = AsNumpyResult(result_ptrs, columns) + + if lazy: + return result + else: + return result.GetValue() - return py_arrays + +class AsNumpyResult(object): + """Future-like class that represents the result of an AsNumpy call. + + Provides AsNumpy with laziness when it comes to triggering the event loop. + + Attributes: + _columns (list): list of the names of the columns returned by + AsNumpy. + _py_arrays (dict): results of the AsNumpy action. The key is the + column name, the value is the NumPy array for that column. + _result_ptrs (dict): results of the AsNumpy action. The key is the + column name, the value is the result pointer for that column. + """ + def __init__(self, result_ptrs, columns): + """Constructs an AsNumpyResult object. + + Parameters: + result_ptrs (dict): results of the AsNumpy action. The key is the + column name, the value is the result pointer for that column. + columns (list): list of the names of the columns returned by + AsNumpy. + """ + + self._result_ptrs = result_ptrs + self._columns = columns + self._py_arrays = None + + def GetValue(self): + """Triggers, if necessary, the event loop to run the Take actions for + the requested columns and produce the NumPy arrays as result. + + Returns: + dict: key is the column name, value is the NumPy array for that + column. + """ + + if self._py_arrays is None: + import numpy + from ROOT.pythonization._rdf_utils import ndarray + + # Convert the C++ vectors to numpy arrays + self._py_arrays = {} + for column in self._columns: + cpp_reference = self._result_ptrs[column].GetValue() + if hasattr(cpp_reference, "__array_interface__"): + tmp = numpy.asarray(cpp_reference) # This adopts the memory of the C++ object. + self._py_arrays[column] = ndarray(tmp, self._result_ptrs[column]) + else: + tmp = numpy.empty(len(cpp_reference), dtype=numpy.object) + for i, x in enumerate(cpp_reference): + tmp[i] = x # This creates only the wrapping of the objects and does not copy. + self._py_arrays[column] = ndarray(tmp, self._result_ptrs[column]) + + return self._py_arrays + + def Merge(self, other): + """ + Merges the numpy arrays in the dictionary of this object with the numpy + arrays in the dictionary of the other object, modifying the attribute of + this object inplace. + + Raises: + - RuntimeError: if either of the method arguments doesn't already + have filled the internal dictionary of numpy arrays. + - ImportError: if the numpy module couldn't be imported. + - ValueError: If the dictionaries of numpy arrays of the two + arguments don't have exactly the same keys. + """ + + if self._py_arrays is None or other._py_arrays is None: + raise RuntimeError("Merging instances of 'AsNumpyResult' failed because either of them didn't compute " + "their result yet. Make sure to call the 'GetValue' method on both objects before " + "trying to merge again.") + + try: + import numpy + except ImportError: + raise ImportError("Failed to import numpy while merging two 'AsNumpyResult' instances.") + + if not self._py_arrays.keys() == other._py_arrays.keys(): + raise ValueError("The two dictionary of numpy arrays have different keys.") + + self._py_arrays = { + key: numpy.concatenate([self._py_arrays[key], + other._py_arrays[key]]) + for key in self._py_arrays + } + + def __getstate__(self): + """ + This function is called during the pickle serialization step. Return the + dictionary of numpy arrays (i.e. the actual result of this `AsNumpy` + call). Other attributes are not needed and the RResultPtr objects are + not serializable at all. + """ + return self.GetValue() + + def __setstate__(self, state): + """ + This function is called during unserialization step. Sets the dictionary + of numpy array of the unserialized object. + """ + self._py_arrays = state def _histo_profile(self, fixed_args, *args): diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/__init__.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/__init__.py index d90e8c2ad0007..124eca4db1f47 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/__init__.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/__init__.py @@ -149,7 +149,7 @@ def rebind_instancemethod(to_class, from_class, func_name): import sys - if sys.version_info > (3, 0): + if sys.version_info >= (3, 0): to_method = from_method else: import new @@ -197,6 +197,10 @@ def pythonize_roofit_class(klass, name): func_new = getattr(python_klass, func_name) import inspect + import sys + + if sys.version_info < (3, 0): + func_new = func_new.__func__ if func_new.__doc__ is None: func_new.__doc__ = func_orig.__doc__ diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabscollection.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabscollection.py index 17440ac0d4d71..7f0c0ce2838a2 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabscollection.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabscollection.py @@ -16,7 +16,7 @@ class RooAbsCollection(object): - """Some member functions of RooAbsCollection that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooAbsCollection that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooAbsCollection::printLatex. For example, the following code is equivalent in PyROOT: \code{.py} # Directly passing a RooCmdArg: @@ -29,7 +29,7 @@ class RooAbsCollection(object): @cpp_signature("RooAbsArg *RooAbsCollection::addClone(const RooAbsArg& var, Bool_t silent=kFALSE) ;") def addClone(self, arg, silent=False): - """The RooAbsCollection::addClone() function is pythonized with the command argument pythonization. + r"""The RooAbsCollection::addClone() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ clonedArg = self._addClone(arg, silent) @@ -37,7 +37,7 @@ def addClone(self, arg, silent=False): @cpp_signature("Bool_t RooAbsCollection::addOwned(RooAbsArg& var, Bool_t silent=kFALSE);") def addOwned(self, arg, silent=False): - """The RooAbsCollection::addOwned() function is pythonized with the command argument pythonization. + r"""The RooAbsCollection::addOwned() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function.""" self._addOwned(arg, silent) SetOwnership(arg, False) @@ -46,10 +46,10 @@ def addOwned(self, arg, silent=False): "RooAbsCollection::printLatex(const RooCmdArg& arg1=RooCmdArg(), const RooCmdArg& arg2=RooCmdArg()," " const RooCmdArg& arg3=RooCmdArg(), const RooCmdArg& arg4=RooCmdArg()," " const RooCmdArg& arg5=RooCmdArg(), const RooCmdArg& arg6=RooCmdArg()," - " const RooCmdArg& arg7=RooCmdArg(), const RooCmdArg& arg8=RooCmdArg()) const ;" + " const RooCmdArg& arg7=RooCmdArg(), const RooCmdArg& arg8=RooCmdArg()) const ;" ) def printLatex(self, *args, **kwargs): - """The RooAbsCollection::printLatex() function is pythonized with the command argument pythonization. + r"""The RooAbsCollection::printLatex() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsCollection.printLatex` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsdata.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsdata.py index 161b2e7e77c9a..f10a7c0c5a480 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsdata.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsdata.py @@ -16,7 +16,7 @@ class RooAbsData(object): - """Some member functions of RooAbsData that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooAbsData that take a RooCmdArg as argument also support keyword arguments. This applies to RooAbsData::plotOn, RooAbsData::createHistogram, RooAbsData::reduce, RooAbsData::statOn. For example, the following code is equivalent in PyROOT: \code{.py} @@ -36,7 +36,7 @@ class RooAbsData(object): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def plotOn(self, *args, **kwargs): - """The RooAbsData::plotOn() function is pythonized with the command argument pythonization. + r"""The RooAbsData::plotOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsData.plotOn` for keyword arguments. @@ -51,7 +51,7 @@ def plotOn(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def createHistogram(self, *args, **kwargs): - """The RooAbsData::createHistogram() function is pythonized with the command argument pythonization. + r"""The RooAbsData::createHistogram() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsData.createHistogram` for keyword arguments. @@ -65,7 +65,7 @@ def createHistogram(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg(),const RooCmdArg& arg8=RooCmdArg()) ;" ) def reduce(self, *args, **kwargs): - """The RooAbsData::reduce() function is pythonized with the command argument pythonization. + r"""The RooAbsData::reduce() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsData.reduce` for keyword arguments. @@ -80,7 +80,7 @@ def reduce(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def statOn(self, *args, **kwargs): - """The RooAbsData::statOn() function is pythonized with the command argument pythonization. + r"""The RooAbsData::statOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsData.statOn` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabspdf.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabspdf.py index 15c1ddbb2333c..275cb2356c331 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabspdf.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabspdf.py @@ -16,7 +16,7 @@ class RooAbsPdf(RooAbsReal): - """Some member functions of RooAbsPdf that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooAbsPdf that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooAbsPdf::fitTo, RooAbsPdf::plotOn, RooAbsPdf::generate, RooAbsPdf::paramOn, RooAbsPdf::createCdf, RooAbsPdf::generateBinned, RooAbsPdf::createChi2, RooAbsPdf::prepareMultiGen and RooAbsPdf::createNLL. For example, the following code is equivalent in PyROOT: @@ -32,7 +32,7 @@ class RooAbsPdf(RooAbsReal): "RooAbsPdf::fitTo(RooAbsData&, const RooCmdArg&, const RooCmdArg&, const RooCmdArg&, const RooCmdArg&, const RooCmdArg&, const RooCmdArg&, const RooCmdArg&, const RooCmdArg&)" ) def fitTo(self, *args, **kwargs): - """The RooAbsPdf::fitTo() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::fitTo() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.fitTo` for keyword arguments. @@ -49,7 +49,7 @@ def fitTo(self, *args, **kwargs): ") const;" ) def plotOn(self, *args, **kwargs): - """The RooAbsPdf::plotOn() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::plotOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.plotOn` for keyword arguments. @@ -63,7 +63,7 @@ def plotOn(self, *args, **kwargs): " const RooCmdArg& arg5=RooCmdArg::none(),const RooCmdArg& arg6=RooCmdArg::none()) ;" ) def generate(self, *args, **kwargs): - """The RooAbsPdf::generate() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::generate() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.generate` for keyword arguments. @@ -78,7 +78,7 @@ def generate(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def paramOn(self, *args, **kwargs): - """The RooAbsPdf::paramOn() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::paramOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.paramOn` for keyword arguments. @@ -91,7 +91,7 @@ def paramOn(self, *args, **kwargs): " const RooCmdArg& arg6=RooCmdArg::none(), const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def createNLL(self, *args, **kwargs): - """The RooAbsPdf::createNLL() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::createNLL() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.createNLL` for keyword arguments. @@ -104,7 +104,7 @@ def createNLL(self, *args, **kwargs): " const RooCmdArg& arg6=RooCmdArg::none(), const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def createChi2(self, *args, **kwargs): - """The RooAbsPdf::createChi2() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::createChi2() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.createChi2` for keyword arguments. @@ -118,7 +118,7 @@ def createChi2(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def createCdf(self, *args, **kwargs): - """The RooAbsPdf::createCdf() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::createCdf() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.createCdf` for keyword arguments. @@ -132,7 +132,7 @@ def createCdf(self, *args, **kwargs): " const RooCmdArg& arg5=RooCmdArg::none(),const RooCmdArg& arg6=RooCmdArg::none()) const;" ) def generateBinned(self, *args, **kwargs): - """The RooAbsPdf::generateBinned() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::generateBinned() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.generateBinned` for keyword arguments. @@ -146,7 +146,7 @@ def generateBinned(self, *args, **kwargs): " const RooCmdArg& arg5=RooCmdArg::none(),const RooCmdArg& arg6=RooCmdArg::none()) ;" ) def prepareMultiGen(self, *args, **kwargs): - """The RooAbsPdf::prepareMultiGen() function is pythonized with the command argument pythonization. + r"""The RooAbsPdf::prepareMultiGen() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsPdf.prepareMultiGen` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreal.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreal.py index 093e96861d185..69e2b0254e27b 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreal.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreal.py @@ -16,7 +16,7 @@ class RooAbsReal(object): - """Some member functions of RooAbsReal that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooAbsReal that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooAbsReal::plotOn, RooAbsReal::createHistogram, RooAbsReal::chi2FitTo, RooAbsReal::createChi2, RooAbsReal::createRunningIntegral and RooAbsReal::createIntegral For example, the following code is equivalent in PyROOT: @@ -38,7 +38,7 @@ class RooAbsReal(object): " const RooCmdArg& arg9=RooCmdArg(), const RooCmdArg& arg10=RooCmdArg()) const ;" ) def plotOn(self, *args, **kwargs): - """The RooAbsReal::plotOn() function is pythonized with the command argument pythonization. + r"""The RooAbsReal::plotOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsReal.plotOn` for keyword arguments. @@ -53,7 +53,7 @@ def plotOn(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def createHistogram(self, *args, **kwargs): - """The RooAbsReal::createHistogram() function is pythonized with the command argument pythonization. + r"""The RooAbsReal::createHistogram() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsReal.createHistogram` for keyword arguments. @@ -67,7 +67,7 @@ def createHistogram(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def createIntegral(self, *args, **kwargs): - """The RooAbsReal::createIntegral() function is pythonized with the command argument pythonization. + r"""The RooAbsReal::createIntegral() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsReal.createIntegral` for keyword arguments. @@ -81,7 +81,7 @@ def createIntegral(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def createRunningIntegral(self, *args, **kwargs): - """The RooAbsReal::createRunningIntegral() function is pythonized with the command argument pythonization. + r"""The RooAbsReal::createRunningIntegral() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsReal.createRunningIntegral` for keyword arguments. @@ -94,7 +94,7 @@ def createRunningIntegral(self, *args, **kwargs): " const RooCmdArg& arg6=RooCmdArg::none(), const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def createChi2(self, *args, **kwargs): - """The RooAbsReal::createChi2() function is pythonized with the command argument pythonization. + r"""The RooAbsReal::createChi2() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsReal.createChi2` for keyword arguments. @@ -107,7 +107,7 @@ def createChi2(self, *args, **kwargs): " const RooCmdArg& arg6=RooCmdArg::none(), const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def chi2FitTo(self, *args, **kwargs): - """The RooAbsReal::chi2FitTo() function is pythonized with the command argument pythonization. + r"""The RooAbsReal::chi2FitTo() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsReal.chi2FitTo` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreallvalue.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreallvalue.py index e2c47320120b4..beeda81d36f3c 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreallvalue.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooabsreallvalue.py @@ -15,7 +15,7 @@ class RooAbsRealLValue(object): - """Some member functions of RooAbsRealLValue that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooAbsRealLValue that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooAbsRealLValue::createHistogram and RooAbsRealLValue::frame. For example, the following code is equivalent in PyROOT: \code{.py} @@ -35,7 +35,7 @@ class RooAbsRealLValue(object): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def createHistogram(self, *args, **kwargs): - """The RooAbsRealLValue::createHistogram() function is pythonized with the command argument pythonization. + r"""The RooAbsRealLValue::createHistogram() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsRealLValue.createHistogram` for keyword arguments. @@ -48,7 +48,7 @@ def createHistogram(self, *args, **kwargs): " const RooCmdArg& arg6=RooCmdArg::none(), const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def frame(self, *args, **kwargs): - """The RooAbsRealLValue::frame() function is pythonized with the command argument pythonization. + r"""The RooAbsRealLValue::frame() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooAbsRealLValue.frame` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roocategory.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roocategory.py index f20e556d4422b..3ae54c785c8ca 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roocategory.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roocategory.py @@ -15,7 +15,7 @@ class RooCategory(object): - """Constructor of RooCategory takes a map as an argument also supports python dictionaries. + r"""Constructor of RooCategory takes a map as an argument also supports python dictionaries. For example, the following code is equivalent in PyROOT: \code{.py} # Default bindings : @@ -30,7 +30,7 @@ class RooCategory(object): @cpp_signature("RooCategory(const char* name, const char* title, const std::map& allowedStates);") def __init__(self, *args, **kwargs): - """The RooCategory constructor is pythonized for converting python dict to std::map. + r"""The RooCategory constructor is pythonized for converting python dict to std::map. The instances in the dict must correspond to the template argument in std::map of the constructor. """ # Redefinition of `RooCategory` constructor for converting python dict to std::map. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roochi2var.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roochi2var.py index a1c6de25bd6e9..c9fce72dddee9 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roochi2var.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roochi2var.py @@ -15,7 +15,7 @@ class RooChi2Var(object): - """Constructor of RooChi2Var takes a RooCmdArg as argument also supports keyword arguments.""" + r"""Constructor of RooChi2Var takes a RooCmdArg as argument also supports keyword arguments.""" @cpp_signature( [ @@ -30,7 +30,7 @@ class RooChi2Var(object): ] ) def __init__(self, *args, **kwargs): - """The RooCategory constructor is pythonized for converting python dict to std::map. + r"""The RooCategory constructor is pythonized for converting python dict to std::map. The keywords must correspond to the CmdArg of the constructor function. """ # Redefinition of `RooChi2Var` constructor for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodatahist.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodatahist.py index ff67e0261f632..1d51a56169438 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodatahist.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodatahist.py @@ -15,7 +15,7 @@ class RooDataHist(object): - """Constructor of RooDataHist takes a RooCmdArg as argument also supports keyword arguments. + r"""Constructor of RooDataHist takes a RooCmdArg as argument also supports keyword arguments. For example, the following code is equivalent in PyROOT: \code{.py} # Directly passing a RooCmdArg: @@ -33,7 +33,7 @@ class RooDataHist(object): ] ) def __init__(self, *args, **kwargs): - """The RooDataHist constructor is pythonized with the command argument pythonization and for converting python dict to std::map. + r"""The RooDataHist constructor is pythonized with the command argument pythonization and for converting python dict to std::map. The keywords must correspond to the CmdArg of the constructor function. The instances in dict must correspond to the template argument in std::map of the constructor. """ diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodataset.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodataset.py index 51a8097e7c3b8..0f4f04c6726f3 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodataset.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodataset.py @@ -15,7 +15,7 @@ class RooDataSet(object): - """Some member functions of RooDataSet that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooDataSet that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooDataSet() constructor and RooDataSet::plotOnXY. For example, the following code is equivalent in PyROOT: \code{.py} @@ -33,7 +33,7 @@ class RooDataSet(object): " const RooCmdArg& arg6=RooCmdArg(),const RooCmdArg& arg7=RooCmdArg(),const RooCmdArg& arg8=RooCmdArg()) ;" ) def __init__(self, *args, **kwargs): - """The RooDataSet constructor is pythonized with the command argument pythonization. + r"""The RooDataSet constructor is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the constructor. """ # Redefinition of `RooDataSet` constructor for keyword arguments. @@ -48,7 +48,7 @@ def __init__(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) const ;" ) def plotOnXY(self, *args, **kwargs): - """The RooDataSet::plotOnXY() function is pythonized with the command argument pythonization. + r"""The RooDataSet::plotOnXY() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooDataSet.plotOnXY` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodecays.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodecays.py index b7802f19c46d7..c885b3708654c 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodecays.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roodecays.py @@ -15,7 +15,7 @@ class RooDecay(object): - """Some constructors of classes like RooDecay, RooBDecay, RooBCPGenDecay, RooBCPEffDecay and RooBMixDecay that take an enum + r"""Some constructors of classes like RooDecay, RooBDecay, RooBCPGenDecay, RooBCPEffDecay and RooBMixDecay that take an enum DecayType as argument also support keyword arguments. For example, the following code is equivalent in PyROOT: \code{.py} @@ -31,7 +31,7 @@ class RooDecay(object): "RooDecay(const char *name, const char *title, RooRealVar& t, RooAbsReal& tau, const RooResolutionModel& model, DecayType type) ;" ) def __init__(self, *args, **kwargs): - """The RooDecay constructor is pythonized with enum values.""" + r"""The RooDecay constructor is pythonized with enum values.""" kwargs = _decaytype_string_to_enum(self, kwargs) self._init(*args, **kwargs) @@ -44,7 +44,7 @@ class RooBDecay(object): " const RooResolutionModel& model, DecayType type);" ) def __init__(self, *args, **kwargs): - """The RooBDecay constructor is pythonized with enum values.""" + r"""The RooBDecay constructor is pythonized with enum values.""" kwargs = _decaytype_string_to_enum(self, kwargs) self._init(*args, **kwargs) @@ -56,7 +56,7 @@ class RooBCPGenDecay(object): " RooAbsReal& delMistag, RooAbsReal& mu, const RooResolutionModel& model, DecayType type=DoubleSided) ;" ) def __init__(self, *args, **kwargs): - """The RooBCPGenDecay constructor is pythonized with enum values.""" + r"""The RooBCPGenDecay constructor is pythonized with enum values.""" kwargs = _decaytype_string_to_enum(self, kwargs) self._init(*args, **kwargs) @@ -69,7 +69,7 @@ class RooBCPEffDecay(object): " const RooResolutionModel& model, DecayType type=DoubleSided) ;" ) def __init__(self, *args, **kwargs): - """The RooBCPEffDecay constructor is pythonized with enum values.""" + r"""The RooBCPEffDecay constructor is pythonized with enum values.""" kwargs = _decaytype_string_to_enum(self, kwargs) self._init(*args, **kwargs) @@ -81,6 +81,6 @@ class RooBMixDecay(object): " RooAbsReal& delMistag, const RooResolutionModel& model, DecayType type=DoubleSided) ;" ) def __init__(self, *args, **kwargs): - """The RooBMixDecay constructor is pythonized with enum values.""" + r"""The RooBMixDecay constructor is pythonized with enum values.""" kwargs = _decaytype_string_to_enum(self, kwargs) self._init(*args, **kwargs) diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roogenfitstudy.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roogenfitstudy.py index b1cfea8cdc720..d1a30b1e5b846 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roogenfitstudy.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roogenfitstudy.py @@ -15,7 +15,7 @@ class RooGenFitStudy(object): - """Some member functions of RooGenFitStudy that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooGenFitStudy that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooGenFitStudy::setGenConfig. """ @@ -26,7 +26,7 @@ class RooGenFitStudy(object): ] ) def setGenConfig(self, *args, **kwargs): - """The RooGenFitStudy::setGenConfig() function is pythonized with the command argument pythonization. + r"""The RooGenFitStudy::setGenConfig() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `RooGenFitStudy.setGenConfig` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooglobalfunc.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooglobalfunc.py index b70198a84a405..faf45adc19225 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooglobalfunc.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooglobalfunc.py @@ -60,7 +60,7 @@ "const RooCmdArg& arg5=RooCmdArg::none(),const RooCmdArg& arg6=RooCmdArg::none()) ;" ) def FitOptions(*args, **kwargs): - """The FitOptions() function is pythonized with the command argument pythonization. + r"""The FitOptions() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `FitOptions` for keyword arguments. @@ -77,7 +77,7 @@ def FitOptions(*args, **kwargs): "const RooCmdArg& arg7=RooCmdArg::none(),const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def Format(*args, **kwargs): - """The Format() function is pythonized with the command argument pythonization. + r"""The Format() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `Format` for keyword arguments. @@ -96,7 +96,7 @@ def Format(*args, **kwargs): "const RooCmdArg& arg5=RooCmdArg::none(), const RooCmdArg& arg6=RooCmdArg::none()) ;" ) def Frame(*args, **kwargs): - """The Frame() function is pythonized with the command argument pythonization. + r"""The Frame() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `Frame` for keyword arguments. @@ -113,7 +113,7 @@ def Frame(*args, **kwargs): "const RooCmdArg& arg7=RooCmdArg::none(),const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def MultiArg(*args, **kwargs): - """The MultiArg() function is pythonized with the command argument pythonization. + r"""The MultiArg() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `MultiArg` for keyword arguments. @@ -125,7 +125,7 @@ def MultiArg(*args, **kwargs): @cpp_signature("YVar(const RooAbsRealLValue& var, const RooCmdArg& arg=RooCmdArg::none()) ;") def YVar(*args, **kwargs): - """The YVar() function is pythonized with the command argument pythonization. + r"""The YVar() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `YVar` for keyword arguments. @@ -140,7 +140,7 @@ def YVar(*args, **kwargs): @cpp_signature("ZVar(const RooAbsRealLValue& var, const RooCmdArg& arg=RooCmdArg::none()) ;") def ZVar(*args, **kwargs): - """The ZVar() function is pythonized with the command argument pythonization. + r"""The ZVar() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `ZVar` for keyword arguments. @@ -155,7 +155,7 @@ def ZVar(*args, **kwargs): @cpp_signature("Slice(std::map const&) ;") def Slice(*args, **kwargs): - """The Slice function is pythonized for converting python dict to std::map. + r"""The Slice function is pythonized for converting python dict to std::map. The keywords must correspond to the CmdArg of the function. The instances in the dict must correspond to the template argument in std::map of the function. """ @@ -178,7 +178,7 @@ def Slice(*args, **kwargs): ] ) def Import(*args, **kwargs): - """The Import function is pythonized for converting python dict to std::map. + r"""The Import function is pythonized for converting python dict to std::map. The keywords must correspond to the CmdArg of the function. The instances in the dict must correspond to the template argument in std::map of the function. """ @@ -195,7 +195,7 @@ def Import(*args, **kwargs): @cpp_signature("Link(const std::map&) ;") def Link(*args, **kwargs): - """The Link function is pythonized for converting python dict to std::map. + r"""The Link function is pythonized for converting python dict to std::map. The keywords must correspond to the CmdArg of the function. The instances in the dict must correspond to the template argument in std::map of the function. """ diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomcstudy.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomcstudy.py index be3913548e126..b52d72941f4c1 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomcstudy.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomcstudy.py @@ -15,7 +15,7 @@ class RooMCStudy(object): - """Some member functions of RooMCStudy that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooMCStudy that take a RooCmdArg as argument also support keyword arguments. So far, this applies to constructor RooMCStudy(), RooMCStudy::plotParamOn, RooMCStudy::plotParam, RooMCStudy::plotNLL, RooMCStudy::plotError and RooMCStudy::plotPull. For example, the following code is equivalent in PyROOT: \code{.py} @@ -34,7 +34,7 @@ class RooMCStudy(object): " const RooCmdArg& arg6=RooCmdArg::none(), const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def __init__(self, *args, **kwargs): - """The RooMCStudy constructor is pythonized with the command argument pythonization. + r"""The RooMCStudy constructor is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the constructor function. """ # Redefinition of `RooMCStudy` constructor for keyword arguments. @@ -48,7 +48,7 @@ def __init__(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def plotParamOn(self, *args, **kwargs): - """The RooMCStudy::plotParamOn() function is pythonized with the command argument pythonization. + r"""The RooMCStudy::plotParamOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function.""" # Redefinition of `RooMCStudy.plotParamOn` for keyword arguments. args, kwargs = _kwargs_to_roocmdargs(*args, **kwargs) @@ -66,7 +66,7 @@ def plotParamOn(self, *args, **kwargs): ] ) def plotParam(self, *args, **kwargs): - """The RooMCStudy::plotParam() function is pythonized with the command argument pythonization. + r"""The RooMCStudy::plotParam() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `RooMCStudy.plotParam` for keyword arguments. @@ -80,7 +80,7 @@ def plotParam(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def plotNLL(self, *args, **kwargs): - """The RooMCStudy::plotNLL() function is pythonized with the command argument pythonization. + r"""The RooMCStudy::plotNLL() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `RooMCStudy.plotNLL` for keyword arguments. @@ -94,7 +94,7 @@ def plotNLL(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def plotError(self, *args, **kwargs): - """The RooMCStudy::plotError() function is pythonized with the command argument pythonization. + r"""The RooMCStudy::plotError() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `RooMCStudy.plotError` for keyword arguments. @@ -108,7 +108,7 @@ def plotError(self, *args, **kwargs): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none()) ;" ) def plotPull(self, *args, **kwargs): - """The RooMCStudy::plotError() function is pythonized with the command argument pythonization. + r"""The RooMCStudy::plotError() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `RooMCStudy.plotPull` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomsgservice.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomsgservice.py index 1ba93ec1a773c..34058463a6c1c 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomsgservice.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roomsgservice.py @@ -15,7 +15,7 @@ class RooMsgService(object): - """Some member functions of RooMsgService that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooMsgService that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooMsgService::addStream. For example, the following code is equivalent in PyROOT: \code{.py} @@ -31,7 +31,7 @@ class RooMsgService(object): " const RooCmdArg& arg4=RooCmdArg(), const RooCmdArg& arg5=RooCmdArg(), const RooCmdArg& arg6=RooCmdArg());" ) def addStream(self, *args, **kwargs): - """The RooMsgService::addStream() function is pythonized with the command argument pythonization. + r"""The RooMsgService::addStream() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the function. """ # Redefinition of `RooMsgService.addStream` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roonllvar.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roonllvar.py index 3ce92f5287b6e..9961003b26d66 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roonllvar.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roonllvar.py @@ -15,7 +15,7 @@ class RooNLLVar(object): - """RooNLLVar() constructor takes a RooCmdArg as argument also supports keyword arguments.""" + r"""RooNLLVar() constructor takes a RooCmdArg as argument also supports keyword arguments.""" @cpp_signature( "RooNLLVar(const char* name, const char* title, RooAbsPdf& pdf, RooAbsData& data," @@ -24,7 +24,7 @@ class RooNLLVar(object): " const RooCmdArg& arg7=RooCmdArg::none(), const RooCmdArg& arg8=RooCmdArg::none(),const RooCmdArg& arg9=RooCmdArg::none()) ;" ) def __init__(self, *args, **kwargs): - """The RooNLLVar constructor is pythonized with the command argument pythonization. + r"""The RooNLLVar constructor is pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the constructor function. """ # Redefinition of `RooNLLVar` constructor for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooprodpdf.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooprodpdf.py index 7c524f5df709c..1ef6051f76f51 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooprodpdf.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooprodpdf.py @@ -15,7 +15,7 @@ class RooProdPdf(object): - """RooProdPdf() constructor takes a RooCmdArg as argument also supports keyword arguments. + r"""RooProdPdf() constructor takes a RooCmdArg as argument also supports keyword arguments. For example, the following code is equivalent in PyROOT: \code{.py} # Directly passing a RooCmdArg: @@ -38,7 +38,7 @@ class RooProdPdf(object): " const RooCmdArg& arg7=RooCmdArg(), const RooCmdArg& arg8=RooCmdArg()) ;" ) def __init__(self, *args, **kwargs): - """The RooProdPdf constructor is pythonized with the command argument pythonization. + r"""The RooProdPdf constructor is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the constructor. """ args, kwargs = _kwargs_to_roocmdargs(*args, **kwargs) diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimultaneous.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimultaneous.py index 2883c2cb0e9f2..c6de907f8196d 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimultaneous.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimultaneous.py @@ -15,7 +15,7 @@ class RooSimultaneous(object): - """Some member functions of RooSimultaneous that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooSimultaneous that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooSimultaneous::plotOn. For example, the following code is equivalent in PyROOT: \code{.py} @@ -36,7 +36,7 @@ class RooSimultaneous(object): " const RooCmdArg& arg9=RooCmdArg(), const RooCmdArg& arg10=RooCmdArg()) const;" ) def plotOn(self, *args, **kwargs): - """The RooSimultaneous::plotOn() function is pythonized with the command argument pythonization. + r"""The RooSimultaneous::plotOn() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooSimultaneous.plotOn` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimwstool.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimwstool.py index bd0c3b57f5f4e..ebb343fd6cf83 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimwstool.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_roosimwstool.py @@ -15,7 +15,7 @@ class RooSimWSTool(object): - """Some member functions of RooSimWSTool that take a RooCmdArg as argument also support keyword arguments. + r"""Some member functions of RooSimWSTool that take a RooCmdArg as argument also support keyword arguments. So far, this applies to RooSimWSTool::build. For example, the following code is equivalent in PyROOT: \code{.py} @@ -34,7 +34,7 @@ class RooSimWSTool(object): " const RooCmdArg& arg5=RooCmdArg::none(),const RooCmdArg& arg6=RooCmdArg::none()) ;" ) def build(self, *args, **kwargs): - """The RooSimWSTool::build() function is pythonized with the command argument pythonization. + r"""The RooSimWSTool::build() function is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the function. """ # Redefinition of `RooSimWSTool.build` for keyword arguments. diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooworkspace.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooworkspace.py index c1891d007159e..2019b0144d345 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooworkspace.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_roofit/_rooworkspace.py @@ -13,7 +13,7 @@ class RooWorkspace(object): - """The RooWorkspace::import function can't be used in PyROOT because `import` is a reserved python keyword. + r"""The RooWorkspace::import function can't be used in PyROOT because `import` is a reserved python keyword. For this reason, an alternative with a capitalized name is provided: \code{.py} workspace.Import(x) @@ -27,7 +27,7 @@ class RooWorkspace(object): " const RooCmdArg& arg7=RooCmdArg(),const RooCmdArg& arg8=RooCmdArg(),const RooCmdArg& arg9=RooCmdArg()) ;" ) def __init__(self, *args, **kwargs): - """The RooWorkspace constructor is pythonized with the command argument pythonization. + r"""The RooWorkspace constructor is pythonized with the command argument pythonization. The keywords must correspond to the CmdArgs of the constructor. """ # Redefinition of `RooWorkspace` constructor for keyword arguments. @@ -56,14 +56,14 @@ def __getitem__(self, key): ] ) def Import(self, *args, **kwargs): - """ + r""" Support the C++ `import()` as `Import()` in python """ return getattr(self, "import")(*args, **kwargs) def RooWorkspace_import(self, *args, **kwargs): - """The RooWorkspace::import function can't be used in PyROOT because `import` is a reserved python keyword. + r"""The RooWorkspace::import function can't be used in PyROOT because `import` is a reserved python keyword. So, Import() is used and pythonized with the command argument pythonization. The keywords must correspond to the CmdArg of the `import()` function. """ diff --git a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_ttree.py b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_ttree.py index f8d142532afa9..86927b060a393 100644 --- a/bindings/pyroot/pythonizations/python/ROOT/pythonization/_ttree.py +++ b/bindings/pyroot/pythonizations/python/ROOT/pythonization/_ttree.py @@ -132,7 +132,6 @@ from libROOTPythonizations import AddBranchAttrSyntax, SetBranchAddressPyz, BranchPyz -import cppyy from ROOT import pythonization # TTree iterator @@ -170,129 +169,6 @@ def _Branch(self, *args): return res -# TTree.AsMatrix functionality -def _TTreeAsMatrix(self, columns=None, exclude=None, dtype="double", return_labels=False): - """Read-out the TTree as a numpy array. - - Note that the reading is performed in multiple threads if the implicit - multi-threading of ROOT is enabled. - - Parameters: - columns: If None return all branches as columns, otherwise specify names in iterable. - exclude: Exclude branches from selection. - dtype: Set return data-type of numpy array. - return_labels: Return additionally to the numpy array the names of the columns. - - Returns: - array(, labels): Numpy array(, labels of columns) - """ - - import warnings - warnings.warn("TTree.AsMatrix is deprecated since v6.24 and will be removed in v6.26. Please use instead RDataFrame.AsNumpy.", FutureWarning) - - # Import numpy lazily - try: - import numpy as np - except: - raise ImportError("Failed to import numpy during call of TTree.AsMatrix.") - - # Check that tree has entries - if self.GetEntries() == 0: - raise Exception("Tree {} has no entries.".format(self.GetName())) - - # Get all columns of the tree if no columns are specified - if columns is None: - columns = [branch.GetName() for branch in self.GetListOfBranches()] - - # Exclude columns - if exclude == None: - exclude = [] - columns = [col for col in columns if not col in exclude] - - if not columns: - raise Exception("Arguments resulted in no selected branches.") - - # Check validity of branches - supported_branch_dtypes = ["Float_t", "Double_t", "Char_t", "UChar_t", "Short_t", "UShort_t", - "Int_t", "UInt_t", "Long64_t", "ULong64_t"] - col_dtypes = [] - invalid_cols_notfound = [] - invalid_cols_dtype = {} - invalid_cols_multipleleaves = {} - invalid_cols_leafname = {} - for col in columns: - # Check that column exists - branch = self.GetBranch(col) - if branch == None: - invalid_cols_notfound.append(col) - continue - - # Check that the branch has only one leaf with the name of the branch - leaves = [leaf.GetName() for leaf in branch.GetListOfLeaves()] - if len(leaves) != 1: - invalid_cols_multipleleaves[col] = len(leaves) - continue - if leaves[0] != col: - invalid_cols_leafname[col] = len(leaves[0]) - continue - - # Check that the leaf of the branch has an arithmetic data-type - col_dtype = self.GetBranch(col).GetLeaf(col).GetTypeName() - col_dtypes.append(col_dtype) - if not col_dtype in supported_branch_dtypes: - invalid_cols_dtype[col] = col_dtype - - exception_template = "Reading of branch {} is not supported ({})." - if invalid_cols_notfound: - raise Exception(exception_template.format(invalid_cols_notfound, "branch not existent")) - if invalid_cols_multipleleaves: - raise Exception(exception_template.format([k for k in invalid_cols_multipleleaves], "branch has multiple leaves")) - if invalid_cols_leafname: - raise Exception(exception_template.format( - [k for k in invalid_cols_leafname], "name of leaf is different from name of branch {}".format( - [invalid_cols_leafname[k] for k in invalid_cols_leafname]))) - if invalid_cols_dtype: - raise Exception(exception_template.format( - [k for k in invalid_cols_dtype], "branch has unsupported data-type {}".format( - [invalid_cols_dtype[k] for k in invalid_cols_dtype]))) - - # Check that given data-type is supported - supported_output_dtypes = ["int", "unsigned int", "long", "unsigned long", "float", "double"] - if not dtype in supported_output_dtypes: - raise Exception("Data-type {} is not supported, select from {}.".format( - dtype, supported_output_dtypes)) - - # Convert columns iterable to std.vector("string") - columns_vector = cppyy.gbl.std.vector["string"](len(columns)) - for i, col in enumerate(columns): - columns_vector[i] = col - - # Allocate memory for the read-out - flat_matrix = cppyy.gbl.std.vector[dtype](self.GetEntries()*len(columns)) - - # Read the tree as flat std.vector(dtype) - tree_ptr = cppyy.gbl.ROOT.Internal.RDF.GetAddress(self) - columns_vector_ptr = cppyy.gbl.ROOT.Internal.RDF.GetAddress(columns_vector) - flat_matrix_ptr = cppyy.gbl.ROOT.Internal.RDF.GetVectorAddress[dtype](flat_matrix) - jit_code = "ROOT::Internal::RDF::TTreeAsFlatMatrixHelper<{dtype}, {col_dtypes}>(*reinterpret_cast({tree_ptr}), *reinterpret_cast* >({flat_matrix_ptr}), *reinterpret_cast* >({columns_vector_ptr}));".format( - col_dtypes = ", ".join(col_dtypes), - dtype = dtype, - tree_ptr = tree_ptr, - flat_matrix_ptr = flat_matrix_ptr, - columns_vector_ptr = columns_vector_ptr) - cppyy.gbl.gInterpreter.Calc(jit_code) - - # Convert the std.vector(dtype) to a numpy array by memory-adoption and - # reshape the flat array to the correct shape of the matrix - flat_matrix_np = np.asarray(flat_matrix) - reshaped_matrix_np = np.reshape(flat_matrix_np, - (int(len(flat_matrix)/len(columns)), len(columns))) - - if return_labels: - return (reshaped_matrix_np, columns) - else: - return reshaped_matrix_np - @pythonization() def pythonize_ttree(klass, name): # Parameters: @@ -326,7 +202,4 @@ def pythonize_ttree(klass, name): klass._OriginalBranch = klass.Branch klass.Branch = _Branch - # AsMatrix - klass.AsMatrix = _TTreeAsMatrix - return True diff --git a/bindings/pyroot/pythonizations/test/CMakeLists.txt b/bindings/pyroot/pythonizations/test/CMakeLists.txt index f8624db875044..f551ef2add21e 100644 --- a/bindings/pyroot/pythonizations/test/CMakeLists.txt +++ b/bindings/pyroot/pythonizations/test/CMakeLists.txt @@ -42,9 +42,6 @@ ROOT_ADD_PYUNITTEST(pyroot_pyz_ttree_branch_attr ttree_branch_attr.py) ROOT_ADD_PYUNITTEST(pyroot_pyz_ttree_iterable ttree_iterable.py) ROOT_ADD_PYUNITTEST(pyroot_pyz_ttree_setbranchaddress ttree_setbranchaddress.py PYTHON_DEPS numpy) ROOT_ADD_PYUNITTEST(pyroot_pyz_ttree_branch ttree_branch.py PYTHON_DEPS numpy) -if (dataframe) - ROOT_ADD_PYUNITTEST(pyroot_pyz_ttree_asmatrix ttree_asmatrix.py PYTHON_DEPS numpy) -endif() # TH1 and subclasses pythonizations ROOT_ADD_PYUNITTEST(pyroot_pyz_th1_operators th1_operators.py) diff --git a/bindings/pyroot/pythonizations/test/dependency_versions.py b/bindings/pyroot/pythonizations/test/dependency_versions.py index 93339f13c986c..7e3b930e4c12d 100644 --- a/bindings/pyroot/pythonizations/test/dependency_versions.py +++ b/bindings/pyroot/pythonizations/test/dependency_versions.py @@ -8,12 +8,13 @@ # Compile list of packages to be ignored in the test ignore = [] -# pyspark is always ignored in this test since we use the build option -# `test_distrdf_pyspark` to check whether the CTest environment is ready for -# distributed RDataFrame tests of the Spark backend. Those tests will be run -# only on the nodes where the build option is enabled and `FindPySpark.cmake` -# was succesfully run during the configuration step. +# Dependencies of distributed RDataFrame are ignored in this test because they +# are checked through specific build options (`test_distrdf_*`). +# The dependencies are checked at configuration time so that we know whether the +# CTest environment would be ready for the tests of the distributed RDF backends ignore.append('pyspark') +ignore.append('dask') +ignore.append('distributed') if sys.version_info[0] == 2 and 'ROOTTEST_IGNORE_NUMBA_PY2' in os.environ or \ sys.version_info[0] == 3 and 'ROOTTEST_IGNORE_NUMBA_PY3' in os.environ: diff --git a/bindings/pyroot/pythonizations/test/ttree_asmatrix.py b/bindings/pyroot/pythonizations/test/ttree_asmatrix.py deleted file mode 100644 index af7fc316da070..0000000000000 --- a/bindings/pyroot/pythonizations/test/ttree_asmatrix.py +++ /dev/null @@ -1,246 +0,0 @@ -import unittest -import ROOT -import numpy as np -from sys import maxsize - - -class TTreeAsMatrix(unittest.TestCase): - is_64bit = True if maxsize > 2**32 else False - - # Helpers - def make_tree(self, *dtypes): - tree = ROOT.TTree("test", "description") - col_names = ["col{}".format(i) for i in range(len(dtypes))] - - col_vars = [] - for dtype in dtypes: - if "F" in dtype: - var = np.empty(1, dtype=np.float32) - elif "D" in dtype: - var = np.empty(1, dtype=np.float64) - elif "I" in dtype: - var = np.empty(1, dtype=np.int32) - elif "i" in dtype: - var = np.empty(1, dtype=np.uint32) - elif "L" in dtype: - if self.is_64bit: - var = np.empty(1, dtype=np.int64) - else: - var = np.empty(1, dtype=np.int32) - elif "l" in dtype: - if self.is_64bit: - var = np.empty(1, dtype=np.uint64) - else: - var = np.empty(1, dtype=np.uint32) - elif "S" in dtype: - var = np.empty(1, dtype=np.int16) - elif "s" in dtype: - var = np.empty(1, dtype=np.uint16) - elif "B" in dtype: - var = np.empty(1, dtype=np.int8) - elif "b" in dtype: - var = np.empty(1, dtype=np.uint8) - elif "O" in dtype: - var = np.empty(1, dtype=np.uint8) - else: - raise Exception( - "Type {} not known to create branch.".format(dtype)) - col_vars.append(var) - - for dtype, name, var in zip(dtypes, col_names, col_vars): - tree.Branch(name, var, name + "/" + dtype) - - reference = [] - for i in range(4): - row = [] - for i_var, var in enumerate(col_vars): - var[0] = (i + i_var + 0.5) * np.power(-1, i) - row.append(var[0]) - reference.append(row) - tree.Fill() - - return tree, reference, dtypes, col_names, col_vars - - def make_example(self, *dtypes): - tree, reference, dtypes, col_names, col_vars = self.make_tree(*dtypes) - matrix_ttree = tree.AsMatrix() - matrix_ref = np.asarray(reference) - return matrix_ttree, matrix_ref - - # Tests - def test_instance(self): - """ - Test instantiation - """ - tree, _, _, col_names, _ = self.make_tree("F", "F") - tree.AsMatrix(col_names) - - def test_return_labels(self): - """ - Test returned labels - """ - tree, _, _, col_names, _ = self.make_tree("F", "F") - matrix, labels = tree.AsMatrix(col_names, return_labels=True) - self.assertEqual(labels, col_names) - - def test_exclude_columns(self): - """ - Test excluding columns - """ - tree, reference, _, _, _ = self.make_tree("F", "F") - matrix_ttree = tree.AsMatrix(exclude=["col0"]) - matrix_ref = np.asarray([x[1] for x in reference]) - for value_ttree, value_ref in zip(matrix_ttree, matrix_ref): - self.assertEqual(value_ttree, value_ref) - - def test_not_supported_dtype(self): - """ - Test error-handling of unsupported data-types - """ - tree, _, _, col_names, _ = self.make_tree("F", "F") - try: - tree.AsMatrix(col_names, dtype="foo") - self.assertFail() - except Exception as exception: - self.assertIn("Data-type foo is not supported, select from", - exception.args[0]) - - def test_not_existent_column(self): - """ - Test if a column does not exist - """ - tree, _, _, col_names, _ = self.make_tree("F", "F") - try: - tree.AsMatrix(["foo"]) - self.assertFail() - except Exception as exception: - self.assertIn("branch not existent", exception.args[0]) - - def test_no_branches_selected(self): - """ - Test if no branch is selected by the requested columns - """ - tree, _, _, col_names, _ = self.make_tree("F", "F") - try: - tree.AsMatrix(columns=["col0"], exclude=["col0"]) - self.assertFail() - except Exception as exception: - self.assertIn("Arguments resulted in no selected branches.", - exception.args[0]) - - def test_shape(self): - """ - Test shape of returned matrix - """ - matrix_ttree, matrix_ref = self.make_example("F", "F") - for i in range(2): - self.assertEqual(matrix_ttree.shape[i], matrix_ref.shape[i]) - - def test_take_all_columns(self): - """ - Test taking all columns - """ - tree, reference, _, _, _ = self.make_tree("F", "F") - matrix_ttree = tree.AsMatrix() - matrix_ref = np.asarray(reference) - for i in range(matrix_ref.shape[0]): - for j in range(matrix_ref.shape[1]): - self.assertEqual(matrix_ttree[i, j], matrix_ref[i, j]) - - def test_values(self): - """ - Test correctness of returned values - """ - matrix_ttree, matrix_ref = self.make_example("F", "F") - for i in range(matrix_ref.shape[0]): - for j in range(matrix_ref.shape[1]): - self.assertEqual(matrix_ttree[i, j], matrix_ref[i, j]) - - def test_zero_entries(self): - """ - Test behaviour for a tree with zero entries - """ - tree = ROOT.TTree("test", "description") - var = np.empty(1, np.float32) - tree.Branch("col", var, "col/F") - try: - tree.AsMatrix(["col"]) - self.assertFail() - except Exception as exception: - self.assertEqual("Tree test has no entries.", exception.args[0]) - - def test_multiple_leaves(self): - """ - Test a column with multiple leaves - """ - tree = ROOT.TTree("test", "description") - var = np.ones(2, np.float32) - tree.Branch("col", var, "sub1/F:sub2/F") - tree.Fill() - try: - tree.AsMatrix(["col"]) - self.assertFail() - except Exception as exception: - self.assertIn("branch has multiple leaves", exception.args[0]) - - def test_dtype_conversion(self): - """ - Test conversion of data-types of the columns to uniform output data-type - """ - numpy_dtype = { - "unsigned int": np.dtype(np.uint32), - "int": np.dtype(np.int32), - "unsigned long": np.dtype(np.uint64), - "long": np.dtype(np.int64), - "float": np.dtype(np.float32), - "double": np.dtype(np.float64) - } - if not self.is_64bit: - numpy_dtype["long"] = np.dtype(np.int32) - numpy_dtype["unsigned long"] = np.dtype(np.uint32) - tree, reference, _, _, _ = self.make_tree("F", "F") - matrix_ref = np.asarray(reference, dtype=np.float64) - for dtype in numpy_dtype: - matrix_ttree = tree.AsMatrix(dtype=dtype) - self.assertEqual(matrix_ttree.dtype.name, numpy_dtype[dtype].name) - - def test_branch_dtypes(self): - """ - Test branches of different data-types - """ - root_dtypes = ["B", "b", "S", "s", "I", "i", "L", "l", "F", "D"] - tree, _, _, col_names, _ = self.make_tree(*root_dtypes) - tree.AsMatrix() - - try: - tree, _, _, col_names, _ = self.make_tree("O") - tree.AsMatrix() - except Exception as exception: - self.assertIn("branch has unsupported data-type ['Bool_t']", - exception.args[0]) - - def test_tchain(self): - """ - Test whether the pythonization works as well for a TChain - """ - tree1 = ROOT.TTree("test1", "description") - tree2 = ROOT.TTree("test2", "description") - var = np.ones(1, np.float32) - tfile = ROOT.TFile("test_ttree_asmatrix.root", "RECREATE") - tree1.Branch("col", var, "col/F") - tree1.Fill() - tree2.Branch("col", var, "col/F") - tree2.Fill() - tree1.Write() - tree2.Write() - tfile.Close() - - chain = ROOT.TChain("chain") - chain.Add("test_ttree_asmatrix.root/test1") - chain.Add("test_ttree_asmatrix.root/test2") - m = chain.AsMatrix(["col"]) - self.assertTrue((m == np.ones((2, 1), dtype=m.dtype)).all()) - - -if __name__ == '__main__': - unittest.main() diff --git a/bindings/pyroot_legacy/CMakeLists.txt b/bindings/pyroot_legacy/CMakeLists.txt index e52fdbed30d5e..0aeb394df051a 100644 --- a/bindings/pyroot_legacy/CMakeLists.txt +++ b/bindings/pyroot_legacy/CMakeLists.txt @@ -106,7 +106,7 @@ endforeach() if (NOT MSVC AND NOT CMAKE_INSTALL_LIBDIR STREQUAL CMAKE_INSTALL_PYTHONDIR) # add a symlink to libPyROOT.so in CMAKE_INSTALL_PYTHONDIR - set(PYROOT_LIBNAME ${CMAKE_SHARED_LIBRARY_PREFIX}PyROOT${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(PYROOT_LIBNAME ${CMAKE_SHARED_LIBRARY_PREFIX}PyROOT.so) file(RELATIVE_PATH PYTHONDIR_TO_LIBDIR "${CMAKE_INSTALL_FULL_PYTHONDIR}" "${CMAKE_INSTALL_FULL_LIBDIR}") install(CODE "file(CREATE_LINK ${PYTHONDIR_TO_LIBDIR}/${PYROOT_LIBNAME} \$ENV{DESTDIR}${CMAKE_INSTALL_FULL_PYTHONDIR}/${PYROOT_LIBNAME} SYMBOLIC)") diff --git a/bindings/r/inc/RExports.h b/bindings/r/inc/RExports.h index 17e84ccc98c97..f5e8ad302dc23 100644 --- a/bindings/r/inc/RExports.h +++ b/bindings/r/inc/RExports.h @@ -30,9 +30,7 @@ #include #include //support for std c++11 classes -// #if __cplusplus > 199711L #include -// #endif //pragma to disable warnings on Rcpp which have //so many noise compiling diff --git a/build/win/bindexplib/bindexplib.cxx b/build/win/bindexplib/bindexplib.cxx index 6c100fffa8ea9..2819d339e4c5b 100644 --- a/build/win/bindexplib/bindexplib.cxx +++ b/build/win/bindexplib/bindexplib.cxx @@ -1,65 +1,171 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +/*------------------------------------------------------------------------- + Portions of this source have been derived from the 'bindexplib' tool + provided by the CERN ROOT Data Analysis Framework project (root.cern.ch). + Permission has been granted by Pere Mato to distribute + this derived work under the CMake license. +-------------------------------------------------------------------------*/ + /* -*---------------------------------------------------------------------- -* Program: dumpexts.exe -* Author: Gordon Chaffee -* -* History: The real functionality of this file was written by -* Matt Pietrek in 1993 in his pedump utility. I've -* modified it to dump the externals in a bunch of object -* files to create a .def file. -* -* Notes: Visual C++ puts an underscore before each exported symbol. -* This file removes them. I don't know if this is a problem -* this other compilers. If _MSC_VER is defined, -* the underscore is removed. If not, it isn't. To get a -* full dump of an object file, use the -f option. This can -* help determine the something that may be different with a -* compiler other than Visual C++. -* ====================================== -* Corrections (Axel 2006-04-04): -* Conversion to C++. Mostly. -* + *---------------------------------------------------------------------- + * Program: dumpexts.exe + * Author: Gordon Chaffee + * + * History: The real functionality of this file was written by + * Matt Pietrek in 1993 in his pedump utility. I've + * modified it to dump the externals in a bunch of object + * files to create a .def file. + * + * Notes: Visual C++ puts an underscore before each exported symbol. + * This file removes them. I don't know if this is a problem + * this other compilers. If _MSC_VER is defined, + * the underscore is removed. If not, it isn't. To get a + * full dump of an object file, use the -f option. This can + * help determine the something that may be different with a + * compiler other than Visual C++. + * ====================================== + * Corrections (Axel 2006-04-04): + * Conversion to C++. Mostly. + * * Extension (Axel 2006-03-15) * As soon as an object file contains an /EXPORT directive (which * is generated by the compiler when a symbol is declared as - * declspec(dllexport)) no to-be-exported symbols are printed, + * __declspec(dllexport) no to-be-exported symbols are printed, * as the linker will see these directives, and if those directives * are present we only export selectively (i.e. we trust the * programmer). * * ====================================== -* ====================================== -* Corrections (Valery Fine 23/02/98): -* -* The "(vector) deleting destructor" MUST not be exported -* To recognize it the following test are introduced: -* "@@UAEPAXI@Z" scalar deleting dtor -* "@@QAEPAXI@Z" vector deleting dtor -* "AEPAXI@Z" vector deleting dtor with thunk adjustor -* ====================================== -* Corrections (Valery Fine 12/02/97): -* -* It created a wrong EXPORTS for the global pointers and constants. -* The Section Header has been involved to discover the missing information -* Now the pointers are correctly supplied supplied with "DATA" descriptor -* the constants with no extra descriptor. -* -* Corrections (Valery Fine 16/09/96): -* -* It didn't work for C++ code with global variables and class definitons -* The DumpExternalObject function has been introduced to generate .DEF file -* -* Author: Valery Fine 16/09/96 (E-mail: fine@vxcern.cern.ch) -*---------------------------------------------------------------------- -*/ - -static char sccsid[] = "@(#) winDumpExts.c 1.2 95/10/03 15:27:34"; - + * ====================================== + * Corrections (Valery Fine 23/02/98): + * + * The "(vector) deleting destructor" MUST not be exported + * To recognize it the following test are introduced: + * "@@UAEPAXI@Z" scalar deleting dtor + * "@@QAEPAXI@Z" vector deleting dtor + * "AEPAXI@Z" vector deleting dtor with thunk adjustor + * ====================================== + * Corrections (Valery Fine 12/02/97): + * + * It created a wrong EXPORTS for the global pointers and constants. + * The Section Header has been involved to discover the missing information + * Now the pointers are correctly supplied with "DATA" descriptor + * the constants with no extra descriptor. + * + * Corrections (Valery Fine 16/09/96): + * + * It didn't work for C++ code with global variables and class definitions + * The DumpExternalObject function has been introduced to generate .DEF + *file + * + * Author: Valery Fine 16/09/96 (E-mail: fine@vxcern.cern.ch) + *---------------------------------------------------------------------- + */ +#include #include + +#include #include #include #include +class bindexplib { +public: + bindexplib() {} + bool AddObjectFile(const char *filename); + void WriteFile(FILE *file); + +private: + std::set Symbols; + std::set DataSymbols; +}; + +#ifndef IMAGE_FILE_MACHINE_ARM +# define IMAGE_FILE_MACHINE_ARM 0x01c0 // ARM Little-Endian +#endif + +#ifndef IMAGE_FILE_MACHINE_THUMB +# define IMAGE_FILE_MACHINE_THUMB 0x01c2 // ARM Thumb/Thumb-2 Little-Endian +#endif + +#ifndef IMAGE_FILE_MACHINE_ARMNT +# define IMAGE_FILE_MACHINE_ARMNT 0x01c4 // ARM Thumb-2 Little-Endian +#endif + +#ifndef IMAGE_FILE_MACHINE_ARM64 +# define IMAGE_FILE_MACHINE_ARM64 0xaa64 // ARM64 Little-Endian +#endif + +typedef struct cmANON_OBJECT_HEADER_BIGOBJ { + /* same as ANON_OBJECT_HEADER_V2 */ + WORD Sig1; // Must be IMAGE_FILE_MACHINE_UNKNOWN + WORD Sig2; // Must be 0xffff + WORD Version; // >= 2 (implies the Flags field is present) + WORD Machine; // Actual machine - IMAGE_FILE_MACHINE_xxx + DWORD TimeDateStamp; + CLSID ClassID; // {D1BAA1C7-BAEE-4ba9-AF20-FAF66AA4DCB8} + DWORD SizeOfData; // Size of data that follows the header + DWORD Flags; // 0x1 -> contains metadata + DWORD MetaDataSize; // Size of CLR metadata + DWORD MetaDataOffset; // Offset of CLR metadata + + /* bigobj specifics */ + DWORD NumberOfSections; // extended from WORD + DWORD PointerToSymbolTable; + DWORD NumberOfSymbols; +} cmANON_OBJECT_HEADER_BIGOBJ; + +typedef struct _cmIMAGE_SYMBOL_EX { + union { + BYTE ShortName[8]; + struct { + DWORD Short; // if 0, use LongName + DWORD Long; // offset into string table + } Name; + DWORD LongName[2]; // PBYTE [2] + } N; + DWORD Value; + LONG SectionNumber; + WORD Type; + BYTE StorageClass; + BYTE NumberOfAuxSymbols; +} cmIMAGE_SYMBOL_EX; +typedef cmIMAGE_SYMBOL_EX UNALIGNED *cmPIMAGE_SYMBOL_EX; + +PIMAGE_SECTION_HEADER GetSectionHeaderOffset( + PIMAGE_FILE_HEADER pImageFileHeader) +{ + return (PIMAGE_SECTION_HEADER)((DWORD_PTR)pImageFileHeader + + IMAGE_SIZEOF_FILE_HEADER + + pImageFileHeader->SizeOfOptionalHeader); +} + +PIMAGE_SECTION_HEADER GetSectionHeaderOffset( + cmANON_OBJECT_HEADER_BIGOBJ *pImageFileHeader) +{ + return (PIMAGE_SECTION_HEADER)((DWORD_PTR)pImageFileHeader + + sizeof(cmANON_OBJECT_HEADER_BIGOBJ)); +} + + +std::wstring ToWide(const std::string &str) +{ + std::wstring wstr; + const int wlength = MultiByteToWideChar( + CP_UTF8, 0, str.data(), int(str.size()), NULL, 0); + if (wlength > 0) { + wchar_t *wdata = new wchar_t[wlength]; + int r = MultiByteToWideChar(CP_UTF8, 0, str.data(), + int(str.size()), wdata, wlength); + if (r > 0) { + wstr = std::wstring(wdata, wlength); + } + delete[] wdata; + } + return wstr; +} + /* *---------------------------------------------------------------------- * GetArgcArgv -- @@ -110,238 +216,12 @@ GetArgcArgv(std::string &s, char **argv) } /* -* The names of the first group of possible symbol table storage classes -*/ -const char *SzStorageClass1[] = { - "NULL","AUTOMATIC","EXTERNAL","STATIC","REGISTER","EXTERNAL_DEF","LABEL", - "UNDEFINED_LABEL","MEMBER_OF_STRUCT","ARGUMENT","STRUCT_TAG", - "MEMBER_OF_UNION","UNION_TAG","TYPE_DEFINITION","UNDEFINED_STATIC", - "ENUM_TAG","MEMBER_OF_ENUM","REGISTER_PARAM","BIT_FIELD" -}; - -/* -* The names of the second group of possible symbol table storage classes -*/ -const char *SzStorageClass2[] = { - "BLOCK","FUNCTION","END_OF_STRUCT","FILE","SECTION","WEAK_EXTERNAL" -}; - -/* -*---------------------------------------------------------------------- -* GetSZStorageClass -- -* -* Given a symbol storage class value, return a descriptive -* ASCII string -*---------------------------------------------------------------------- -*/ -PCSTR -GetSZStorageClass(BYTE storageClass) -{ - if ( storageClass <= IMAGE_SYM_CLASS_BIT_FIELD ) - return SzStorageClass1[storageClass]; - else if ( (storageClass >= IMAGE_SYM_CLASS_BLOCK) - && (storageClass <= IMAGE_SYM_CLASS_WEAK_EXTERNAL) ) - return SzStorageClass2[storageClass-IMAGE_SYM_CLASS_BLOCK]; - else - return "???"; -} - -void AddHex(std::string& buf, long val, bool caps=false) -{ - buf += "0x"; - int len=buf.length(); - while (val) { - char hex = (char)(val & 16); - val = val >> 4; - if (hex<10) hex+='0'; - else if (caps) hex+='A'-10; - else hex+='a'-10; - buf.insert(len, hex, 1); - } -} - -/* -*---------------------------------------------------------------------- -* GetSectionName -- -* -* Used by DumpSymbolTable, it gives meaningful names to -* the non-normal section number. -* -* Results: -* A name is returned in buffer -*---------------------------------------------------------------------- -*/ -void -GetSectionName(PIMAGE_SYMBOL pSymbolTable, std::string& buffer) -{ - DWORD section; - - section = pSymbolTable->SectionNumber; - - switch ( (SHORT)section ) - { - case IMAGE_SYM_UNDEFINED: if (pSymbolTable->Value) buffer += "COMM"; else buffer += "UNDEF"; break; - case IMAGE_SYM_ABSOLUTE: buffer += "ABS "; break; - case IMAGE_SYM_DEBUG: buffer += "DEBUG"; break; - default: AddHex(buffer, section, true); - } -} - -/* -*---------------------------------------------------------------------- -* GetSectionCharacteristics -- -* -* Converts the Characteristics field of IMAGE_SECTION_HEADER -* to print. -* -* Results: -* A definiton of the section symbol type -*---------------------------------------------------------------------- -*/ -void -GetSectionCharacteristics(PIMAGE_SECTION_HEADER pSectionHeaders, int nSectNum, std::string &buffer) -{ - DWORD SectChar; - std::string TempBuf; - buffer.clear(); - if (nSectNum > 0) { - SectChar = pSectionHeaders[nSectNum-1].Characteristics; - - buffer = " "; - AddHex(buffer, SectChar); - if (SectChar & IMAGE_SCN_CNT_CODE) buffer += " Code"; - else if (SectChar & IMAGE_SCN_CNT_INITIALIZED_DATA) buffer += " Init. data"; - else if (SectChar & IMAGE_SCN_CNT_UNINITIALIZED_DATA ) buffer += " UnInit data"; - else buffer += " Unknow type"; - - if (SectChar & IMAGE_SCN_MEM_READ) { - buffer += " Read"; - if (SectChar & IMAGE_SCN_MEM_WRITE) - buffer += " and Write"; - else buffer += " only"; - } - else if (SectChar & IMAGE_SCN_MEM_WRITE) - buffer +=" Write only"; - - } -} - -/* -*---------------------------------------------------------------------- -* DumpSymbolTable -- -* -* Dumps a COFF symbol table from an EXE or OBJ. We only use -* it to dump tables from OBJs. -*---------------------------------------------------------------------- -*/ -void -DumpSymbolTable(PIMAGE_SYMBOL pSymbolTable, PIMAGE_SECTION_HEADER pSectionHeaders, FILE *fout, unsigned cSymbols) -{ - unsigned i; - PSTR stringTable; - std::string sectionName; - std::string sectionCharacter; - int iSectNum; - - fprintf(fout, "Symbol Table - %X entries (* = auxillary symbol)\n", - cSymbols); - - fprintf(fout, - "Indx Name Value Section cAux Type Storage Character\n" - "---- -------------------- -------- ---------- ----- ------- -------- ---------\n"); - - /* - * The string table apparently starts right after the symbol table - */ - stringTable = (PSTR)&pSymbolTable[cSymbols]; - - for ( i=0; i < cSymbols; i++ ) { - fprintf(fout, "%04X ", i); - if ( pSymbolTable->N.Name.Short != 0 ) - fprintf(fout, "%-20.8s", pSymbolTable->N.ShortName); - else - fprintf(fout, "%-20s", stringTable + pSymbolTable->N.Name.Long); - - fprintf(fout, " %08X", pSymbolTable->Value); - - iSectNum = pSymbolTable->SectionNumber; - GetSectionName(pSymbolTable, sectionName); - fprintf(fout, " sect:%s aux:%X type:%02X st:%s", - sectionName.c_str(), - pSymbolTable->NumberOfAuxSymbols, - pSymbolTable->Type, - GetSZStorageClass(pSymbolTable->StorageClass) ); - - GetSectionCharacteristics(pSectionHeaders,iSectNum,sectionCharacter); - fprintf(fout," hc: %s \n",sectionCharacter.c_str()); -#if 0 - if ( pSymbolTable->NumberOfAuxSymbols ) - DumpAuxSymbols(pSymbolTable); -#endif - - /* - * Take into account any aux symbols - */ - i += pSymbolTable->NumberOfAuxSymbols; - pSymbolTable += pSymbolTable->NumberOfAuxSymbols; - pSymbolTable++; - } -} - -/* -*---------------------------------------------------------------------- -* DumpExternals -- -* -* Dumps a COFF symbol table from an EXE or OBJ. We only use -* it to dump tables from OBJs. -*---------------------------------------------------------------------- -*/ -void -DumpExternals(PIMAGE_SYMBOL pSymbolTable, FILE *fout, unsigned cSymbols) + * Utility func, strstr with size + */ +const char *StrNStr(const char *start, const char *find, size_t &size) { - unsigned i; - PSTR stringTable; - std::string symbol; - - /* - * The string table apparently starts right after the symbol table - */ - stringTable = (PSTR)&pSymbolTable[cSymbols]; - - for ( i=0; i < cSymbols; i++ ) { - if (pSymbolTable->SectionNumber > 0 && pSymbolTable->Type == 0x20) { - if (pSymbolTable->StorageClass == IMAGE_SYM_CLASS_EXTERNAL) { - if (pSymbolTable->N.Name.Short != 0) { - symbol = ""; - symbol.insert(0, (const char *)(pSymbolTable->N.ShortName), 8); - } else { - symbol = stringTable + pSymbolTable->N.Name.Long; - } - std::string::size_type posAt = symbol.find('@'); - if (posAt != std::string::npos) symbol.erase(posAt); -#ifndef _MSC_VER - fprintf(fout, "\t%s\n", symbol.c_str()); -#else - fprintf(fout, "\t%s\n", symbol.c_str()+1); -#endif - } - } - - /* - * Take into account any aux symbols - */ - i += pSymbolTable->NumberOfAuxSymbols; - pSymbolTable += pSymbolTable->NumberOfAuxSymbols; - pSymbolTable++; - } -} - -/* -+ * Utility func, strstr with size -+ */ -const char* StrNStr(const char* start, const char* find, size_t &size) { - int len; - const char* hint; + size_t len; + const char *hint; if (!start || !find || !size) { size = 0; @@ -349,7 +229,7 @@ const char* StrNStr(const char* start, const char* find, size_t &size) { } len = strlen(find); - while (hint = (const char*) memchr(start, find[0], size-len+1)) { + while ((hint = (const char *)memchr(start, find[0], size - len + 1))) { size -= (hint - start); if (!strncmp(hint, find, len)) return hint; @@ -360,226 +240,165 @@ const char* StrNStr(const char* start, const char* find, size_t &size) { return 0; } -/* - *---------------------------------------------------------------------- - * HaveExportedObjects -- - * - * Returns >0 if export directives (declspec(dllexport)) exist. - * - *---------------------------------------------------------------------- - */ -int -HaveExportedObjects(PIMAGE_FILE_HEADER pImageFileHeader, PIMAGE_SECTION_HEADER pSectionHeaders, FILE *fout) -{ - static int fImportFlag = 0; /* The status is nor defined yet */ - WORD i; - size_t size; - char foundExports; - const char * rawdata; - - PIMAGE_SECTION_HEADER pDirectivesSectionHeader; - - if (fImportFlag) return 1; - - i = 0; - foundExports = 0; - pDirectivesSectionHeader = 0; - for(i = 0; i < pImageFileHeader->NumberOfSections && !pDirectivesSectionHeader; i++) - if (!strncmp((const char*)&pSectionHeaders[i].Name[0], ".drectve",8)) - pDirectivesSectionHeader = &pSectionHeaders[i]; - if (!pDirectivesSectionHeader) return 0; - - rawdata=(const char*)pImageFileHeader+pDirectivesSectionHeader->PointerToRawData; - if (!pDirectivesSectionHeader->PointerToRawData || !rawdata) return 0; - - size = pDirectivesSectionHeader->SizeOfRawData; - const char* posImportFlag = rawdata; - while ((posImportFlag = StrNStr(posImportFlag, " /EXPORT:", size))) { - const char* lookingForDict = posImportFlag + 9; - if (!strncmp(lookingForDict, "_G__cpp_",8) || - !strncmp(lookingForDict, "_G__set_cpp_",12)) { - posImportFlag = lookingForDict; - continue; - } - - const char* lookingForDATA = posImportFlag + 9; - while (*(++lookingForDATA) && *lookingForDATA != ' '); - lookingForDATA -= 5; - // ignore DATA exports - if (strncmp(lookingForDATA, ",DATA", 5)) break; - posImportFlag = lookingForDATA + 5; - } - fImportFlag = (int)posImportFlag; - return fImportFlag; -} - - - -/* - *---------------------------------------------------------------------- -* DumpExternalsObjects -- -* -* Dumps a COFF symbol table from an EXE or OBJ. We only use -* it to dump tables from OBJs. -*---------------------------------------------------------------------- -*/ -void -DumpExternalsObjects(PIMAGE_SYMBOL pSymbolTable, PIMAGE_SECTION_HEADER pSectionHeaders, - FILE *fout, unsigned cSymbols, int fort) -{ - unsigned i; - PSTR stringTable; - std::string symbol; - DWORD SectChar; - static int fImportFlag = -1; /* The status is nor defined yet */ +template < + // cmANON_OBJECT_HEADER_BIGOBJ or IMAGE_FILE_HEADER + class ObjectHeaderType, + // cmPIMAGE_SYMBOL_EX or PIMAGE_SYMBOL + class SymbolTableType > +class DumpSymbols { +public: + /* + *---------------------------------------------------------------------- + * Constructor -- + * + * Initialize variables from pointer to object header. + * + *---------------------------------------------------------------------- + */ + + DumpSymbols(ObjectHeaderType *ih, std::set &symbols, + std::set &dataSymbols, bool isI386) + : Symbols(symbols) + , DataSymbols(dataSymbols) + { + this->ObjectImageHeader = ih; + this->SymbolTable = + (SymbolTableType *)((DWORD_PTR)this->ObjectImageHeader + + this->ObjectImageHeader->PointerToSymbolTable); + this->SectionHeaders = GetSectionHeaderOffset(this->ObjectImageHeader); + this->SymbolCount = this->ObjectImageHeader->NumberOfSymbols; + this->IsI386 = isI386; + } /* - * The string table apparently starts right after the symbol table - */ - stringTable = (PSTR)&pSymbolTable[cSymbols]; - - for ( i=0; i < cSymbols; i++ ) { - if (pSymbolTable->SectionNumber > 0 && ( pSymbolTable->Type == 0x20 || pSymbolTable->Type == 0x0)) { - if (pSymbolTable->StorageClass == IMAGE_SYM_CLASS_EXTERNAL) { - /* - * The name of the Function entry points - */ - if (pSymbolTable->N.Name.Short != 0) { - symbol = ""; - symbol.insert(0, (const char *)pSymbolTable->N.ShortName, 8); - } else { - symbol = stringTable + pSymbolTable->N.Name.Long; - } + *---------------------------------------------------------------------- + * DumpObjFile -- + * + * Dump an object file's exported symbols. + *---------------------------------------------------------------------- + */ + void DumpObjFile() + { + this->DumpExternalsObjects(); + } - while (isspace(symbol[0])) symbol.erase(0,1); -#ifdef _MSC_VER - if (symbol[0] == '_') symbol.erase(0,1); - if (fort) { - std::string::size_type posAt = symbol.find('@'); - if (posAt != std::string::npos) symbol.erase(posAt); - } -#endif - if (fImportFlag) { - fImportFlag = 0; - fprintf(fout,"EXPORTS \n"); - } - /* - Check whether it is "Scalar deleting destructor" and - "Vector deleting destructor" - */ - /* - if (!strstr(s,"@@UAEPAXI@Z") && !strstr(s,"@@QAEPAXI@Z") && - !strstr(s,"@AEPAXI@Z") && !strstr(s,"AEPAXI@Z") && - !strstr(s,"real@")) - */ - const char *scalarPrefix = "??_G"; - const char *vectorPrefix = "??_E"; - if (symbol.compare(0, 4, scalarPrefix) && - symbol.compare(0, 4, vectorPrefix) && - symbol.find("real@") == std::string::npos) - { - SectChar = pSectionHeaders[pSymbolTable->SectionNumber-1].Characteristics; - if (!pSymbolTable->Type && (SectChar & IMAGE_SCN_MEM_WRITE)) { - // Read only (i.e. constants) must be excluded - fprintf(fout, "\t%s \t DATA\n", symbol.c_str()); + /* + *---------------------------------------------------------------------- + * DumpExternalsObjects -- + * + * Dumps a COFF symbol table from an OBJ. + *---------------------------------------------------------------------- + */ + void DumpExternalsObjects() + { + unsigned i; + PSTR stringTable; + std::string symbol; + DWORD SectChar; + /* + * The string table apparently starts right after the symbol table + */ + stringTable = (PSTR) & this->SymbolTable[this->SymbolCount]; + SymbolTableType *pSymbolTable = this->SymbolTable; + for (i = 0; i < this->SymbolCount; i++) { + if (pSymbolTable->SectionNumber > 0 && + (pSymbolTable->Type == 0x20 || pSymbolTable->Type == 0x0)) { + if (pSymbolTable->StorageClass == IMAGE_SYM_CLASS_EXTERNAL) { + /* + * The name of the Function entry points + */ + if (pSymbolTable->N.Name.Short != 0) { + symbol.clear(); + symbol.insert(0, (const char *)pSymbolTable->N.ShortName, 8); } else { - if ( pSymbolTable->Type || !(SectChar & IMAGE_SCN_MEM_READ)) { - fprintf(fout, "\t%s\n", symbol.c_str()); - } else { - // printf(" strange symbol: %s \n",s); + symbol = stringTable + pSymbolTable->N.Name.Long; + } + + // clear out any leading spaces + while (isspace(symbol[0])) + symbol.erase(0, 1); + // if it starts with _ and has an @ then it is a __cdecl + // so remove the @ stuff for the export + if (symbol[0] == '_') { + std::string::size_type posAt = symbol.find('@'); + if (posAt != std::string::npos) { + symbol.erase(posAt); + } + } + // For i386 builds we need to remove _ + if (this->IsI386 && symbol[0] == '_') { + symbol.erase(0, 1); + } + + // Check whether it is "Scalar deleting destructor" and "Vector + // deleting destructor" + // if scalarPrefix and vectorPrefix are not found then print + // the symbol + const char *scalarPrefix = "??_G"; + const char *vectorPrefix = "??_E"; + // The original code had a check for + // symbol.find("real@") == std::string::npos) + // but this disallows member functions with the name "real". + if (symbol.compare(0, 4, scalarPrefix) && + symbol.compare(0, 4, vectorPrefix)) { + SectChar = this->SectionHeaders[pSymbolTable->SectionNumber - 1] + .Characteristics; + // skip symbols containing a dot + if (symbol.find('.') == std::string::npos) { + if (!pSymbolTable->Type && (SectChar & IMAGE_SCN_MEM_WRITE)) { + // Read only (i.e. constants) must be excluded + this->DataSymbols.insert(symbol); + } else { + if (pSymbolTable->Type || !(SectChar & IMAGE_SCN_MEM_READ) || + (SectChar & IMAGE_SCN_MEM_EXECUTE)) { + this->Symbols.insert(symbol); + } + } } } } } - } - else if (pSymbolTable->SectionNumber == IMAGE_SYM_UNDEFINED && !pSymbolTable->Type && 0){ + /* - * The IMPORT global variable entry points - */ - if (pSymbolTable->StorageClass == IMAGE_SYM_CLASS_EXTERNAL) { - symbol = stringTable + pSymbolTable->N.Name.Long; - while (isspace(symbol[0])) symbol.erase(0,1); - if (symbol[0] == '_') symbol.erase(0,1); - if (!fImportFlag) { - fImportFlag = 1; - fprintf(fout,"IMPORTS \n"); - } - fprintf(fout, "\t%s DATA \n", symbol.c_str()+1); - } + * Take into account any aux symbols + */ + i += pSymbolTable->NumberOfAuxSymbols; + pSymbolTable += pSymbolTable->NumberOfAuxSymbols; + pSymbolTable++; } - - /* - * Take into account any aux symbols - */ - i += pSymbolTable->NumberOfAuxSymbols; - pSymbolTable += pSymbolTable->NumberOfAuxSymbols; - pSymbolTable++; } -} -/* -*---------------------------------------------------------------------- -* DumpObjFile -- -* -* Dump an object file--either a full listing or just the exported -* symbols. -*---------------------------------------------------------------------- -*/ -void -DumpObjFile(PIMAGE_FILE_HEADER pImageFileHeader, FILE *fout, int full, int fort) -{ - PIMAGE_SYMBOL PCOFFSymbolTable; - PIMAGE_SECTION_HEADER PCOFFSectionHeaders; - DWORD COFFSymbolCount; - - PCOFFSymbolTable = (PIMAGE_SYMBOL) - ((DWORD)pImageFileHeader + pImageFileHeader->PointerToSymbolTable); - COFFSymbolCount = pImageFileHeader->NumberOfSymbols; - - PCOFFSectionHeaders = (PIMAGE_SECTION_HEADER) - ((DWORD)pImageFileHeader + - IMAGE_SIZEOF_FILE_HEADER + - pImageFileHeader->SizeOfOptionalHeader); - - - if (full) { - DumpSymbolTable(PCOFFSymbolTable, PCOFFSectionHeaders, fout, COFFSymbolCount); - } else { - int haveExports = HaveExportedObjects(pImageFileHeader, PCOFFSectionHeaders, fout); - if (!haveExports) - DumpExternalsObjects(PCOFFSymbolTable, PCOFFSectionHeaders, fout, COFFSymbolCount, fort); - } -} +private: + std::set &Symbols; + std::set &DataSymbols; + DWORD_PTR SymbolCount; + PIMAGE_SECTION_HEADER SectionHeaders; + ObjectHeaderType *ObjectImageHeader; + SymbolTableType *SymbolTable; + bool IsI386; +}; -/* -*---------------------------------------------------------------------- -* DumpFile -- -* -* Open up a file, memory map it, and call the appropriate -* dumping routine -*---------------------------------------------------------------------- -*/ -void -DumpFile(LPSTR filename, FILE *fout, int full, int fort) +bool DumpFile(const char *filename, std::set &symbols, + std::set &dataSymbols) { HANDLE hFile; HANDLE hFileMapping; LPVOID lpFileBase; - PIMAGE_DOS_HEADER dosHeader; - hFile = CreateFile(filename, GENERIC_READ, FILE_SHARE_READ, NULL, - OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); + hFile = CreateFileW(ToWide(filename).c_str(), GENERIC_READ, + FILE_SHARE_READ, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, 0); if (hFile == INVALID_HANDLE_VALUE) { - fprintf(stderr, "Couldn't open file with CreateFile()\n"); - return; + fprintf(stderr, "Couldn't open file '%s' with CreateFile()\n", filename); + return false; } hFileMapping = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL); if (hFileMapping == 0) { CloseHandle(hFile); fprintf(stderr, "Couldn't open file mapping with CreateFileMapping()\n"); - return; + return false; } lpFileBase = MapViewOfFile(hFileMapping, FILE_MAP_READ, 0, 0, 0); @@ -587,34 +406,70 @@ DumpFile(LPSTR filename, FILE *fout, int full, int fort) CloseHandle(hFileMapping); CloseHandle(hFile); fprintf(stderr, "Couldn't map view of file with MapViewOfFile()\n"); - return; + return false; } - dosHeader = (PIMAGE_DOS_HEADER)lpFileBase; + const PIMAGE_DOS_HEADER dosHeader = (PIMAGE_DOS_HEADER)lpFileBase; if (dosHeader->e_magic == IMAGE_DOS_SIGNATURE) { -#if 0 - DumpExeFile( dosHeader ); -#else fprintf(stderr, "File is an executable. I don't dump those.\n"); - return; -#endif - } - /* Does it look like a i386 COFF OBJ file??? */ - else if ((dosHeader->e_magic == 0x014C) && (dosHeader->e_sp == 0)) { - /* - * The two tests above aren't what they look like. They're - * really checking for IMAGE_FILE_HEADER.Machine == i386 (0x14C) - * and IMAGE_FILE_HEADER.SizeOfOptionalHeader == 0; - */ - DumpObjFile((PIMAGE_FILE_HEADER) lpFileBase, fout, full, fort); + return false; } else { - printf("unrecognized file format\n"); + const PIMAGE_FILE_HEADER imageHeader = (PIMAGE_FILE_HEADER)lpFileBase; + /* Does it look like a COFF OBJ file??? */ + if (((imageHeader->Machine == IMAGE_FILE_MACHINE_I386) || + (imageHeader->Machine == IMAGE_FILE_MACHINE_AMD64) || + (imageHeader->Machine == IMAGE_FILE_MACHINE_ARM) || + (imageHeader->Machine == IMAGE_FILE_MACHINE_ARMNT) || + (imageHeader->Machine == IMAGE_FILE_MACHINE_ARM64)) && + (imageHeader->Characteristics == 0)) { + /* + * The tests above are checking for IMAGE_FILE_HEADER.Machine + * if it contains supported machine formats (currently ARM and x86) + * and IMAGE_FILE_HEADER.Characteristics == 0 indicating that + * this is not linked COFF OBJ file; + */ + DumpSymbols symbolDumper( + (PIMAGE_FILE_HEADER)lpFileBase, symbols, dataSymbols, + (imageHeader->Machine == IMAGE_FILE_MACHINE_I386)); + symbolDumper.DumpObjFile(); + } else { + // check for /bigobj format + cmANON_OBJECT_HEADER_BIGOBJ *h = + (cmANON_OBJECT_HEADER_BIGOBJ *)lpFileBase; + if (h->Sig1 == 0x0 && h->Sig2 == 0xffff) { + DumpSymbols + symbolDumper((cmANON_OBJECT_HEADER_BIGOBJ *)lpFileBase, symbols, + dataSymbols, (h->Machine == IMAGE_FILE_MACHINE_I386)); + symbolDumper.DumpObjFile(); + } else { + printf("unrecognized file format in '%s'\n", filename); + return false; + } + } } UnmapViewOfFile(lpFileBase); CloseHandle(hFileMapping); CloseHandle(hFile); + return true; } +bool bindexplib::AddObjectFile(const char *filename) +{ + return DumpFile(filename, this->Symbols, this->DataSymbols); +} + +void bindexplib::WriteFile(FILE *file) +{ + fprintf(file, "EXPORTS \n"); + for (std::string const &ds : this->DataSymbols) { + fprintf(file, "\t%s \t DATA\n", ds.c_str()); + } + for (std::string const &s : this->Symbols) { + fprintf(file, "\t%s\n", s.c_str()); + } +} + + void main(int argc, char **argv) { @@ -658,7 +513,7 @@ main(int argc, char **argv) fout = fopen(outfile, "w+"); if (fout == NULL) { fprintf(stderr, "Unable to open \'%s\' for writing:\n", - argv[arg]); + argv[arg]); perror(""); exit(1); } @@ -673,7 +528,7 @@ main(int argc, char **argv) goto Usage; } fprintf(fout, "LIBRARY %s\n", dllname); -#ifndef _X86_ +#if 0 //ndef _X86_ fprintf(fout, "CODE PRELOAD MOVEABLE DISCARDABLE\n"); fprintf(fout, "DATA PRELOAD MOVEABLE MULTIPLE\n\n"); #endif @@ -686,7 +541,7 @@ main(int argc, char **argv) std::ifstream fargs(&argv[arg][1]); if (!fargs) { fprintf(stderr, "Unable to open \'%s\' for reading:\n", - argv[arg]); + argv[arg]); perror(""); exit(1); } @@ -706,13 +561,13 @@ main(int argc, char **argv) /* * Argument can contain the wildcard names */ - SearchFile = FindFirstFile(argv[arg],&FindFileData); - if (SearchFile == INVALID_HANDLE_VALUE){ + bindexplib deffile; + SearchFile = FindFirstFile(argv[arg], &FindFileData); + if (SearchFile == INVALID_HANDLE_VALUE) { fprintf(stderr, "Unable to find \'%s\' for reading:\n", - argv[arg]); + argv[arg]); exit(1); - } - else { + } else { /* * Since WIN32_FIND_DATA has no path information one has to extract it oneself. */ @@ -720,16 +575,17 @@ main(int argc, char **argv) TCHAR path[2048]; int i = strlen(filename); i--; - while( filename[i] != '\\' && filename[i] != '/' && i >=0) i--; - do - { - if (i >= 0) strncpy( path, filename, i+1); /* Generate the 'path' info */ - path[i+1] = '\0'; - DumpFile(strcat(path, FindFileData.cFileName), fout, full, fort); - } while (FindNextFile(SearchFile,&FindFileData)); - + while (filename[i] != '\\' && filename[i] != '/' && i >= 0) i--; + do { + if (i >= 0) strncpy(path, filename, i + 1); /* Generate the 'path' info */ + path[i + 1] = '\0'; + if (!deffile.AddObjectFile(strcat(path, FindFileData.cFileName))) { + break; + } + } while (FindNextFile(SearchFile, &FindFileData)); FindClose(SearchFile); + deffile.WriteFile(fout); } } exit(0); diff --git a/builtins/davix/CMakeLists.txt b/builtins/davix/CMakeLists.txt index e56369729aa0d..86b645bec5933 100644 --- a/builtins/davix/CMakeLists.txt +++ b/builtins/davix/CMakeLists.txt @@ -79,7 +79,7 @@ set(DAVIX_LIBRARY ${DAVIX_PREFIX}/lib/${DAVIX_LIBNAME} CACHE INTERNAL "" FORCE) set(DAVIX_LIBRARIES ${DAVIX_LIBRARIES} CACHE INTERNAL "" FORCE) add_library(davix INTERFACE) -target_include_directories(davix INTERFACE $) +target_include_directories(davix SYSTEM INTERFACE $) target_link_libraries(davix INTERFACE $) add_dependencies(davix DAVIX) diff --git a/builtins/nlohmann/CMakeLists.txt b/builtins/nlohmann/CMakeLists.txt index 92aff1c02639a..b085def220a32 100644 --- a/builtins/nlohmann/CMakeLists.txt +++ b/builtins/nlohmann/CMakeLists.txt @@ -17,18 +17,10 @@ add_custom_command( COMMENT "Copying nlohmann/json.hpp header to ${CMAKE_BINARY_DIR}/include" DEPENDS ${CMAKE_SOURCE_DIR}/builtins/nlohmann/json.hpp) -add_custom_command( - OUTPUT ${CMAKE_BINARY_DIR}/include/nlohmann/json_fwd.hpp - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/builtins/nlohmann/json_fwd.hpp ${CMAKE_BINARY_DIR}/include/nlohmann/json_fwd.hpp - COMMENT "Copying nlohmann/json_fwd.hpp header to ${CMAKE_BINARY_DIR}/include" - DEPENDS ${CMAKE_SOURCE_DIR}/builtins/nlohmann/json_fwd.hpp) - -add_custom_target(builtin_nlohmann_json_incl - DEPENDS ${CMAKE_BINARY_DIR}/include/nlohmann/json.hpp ${CMAKE_BINARY_DIR}/include/nlohmann/json_fwd.hpp) +add_custom_target(builtin_nlohmann_json_incl DEPENDS ${CMAKE_BINARY_DIR}/include/nlohmann/json.hpp) set_property(GLOBAL APPEND PROPERTY ROOT_HEADER_TARGETS builtin_nlohmann_json_incl) -install(FILES ${CMAKE_SOURCE_DIR}/builtins/nlohmann/json.hpp ${CMAKE_SOURCE_DIR}/builtins/nlohmann/json_fwd.hpp - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nlohmann/) +install(FILES ${CMAKE_SOURCE_DIR}/builtins/nlohmann/json.hpp DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nlohmann/) diff --git a/builtins/nlohmann/json_fwd.hpp b/builtins/nlohmann/json_fwd.hpp deleted file mode 100644 index 332227c1ba280..0000000000000 --- a/builtins/nlohmann/json_fwd.hpp +++ /dev/null @@ -1,78 +0,0 @@ -#ifndef INCLUDE_NLOHMANN_JSON_FWD_HPP_ -#define INCLUDE_NLOHMANN_JSON_FWD_HPP_ - -#include // int64_t, uint64_t -#include // map -#include // allocator -#include // string -#include // vector - -/*! -@brief namespace for Niels Lohmann -@see https://github.com/nlohmann -@since version 1.0.0 -*/ -namespace nlohmann -{ -/*! -@brief default JSONSerializer template argument - -This serializer ignores the template arguments and uses ADL -([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) -for serialization. -*/ -template -struct adl_serializer; - -template class ObjectType = - std::map, - template class ArrayType = std::vector, - class StringType = std::string, class BooleanType = bool, - class NumberIntegerType = std::int64_t, - class NumberUnsignedType = std::uint64_t, - class NumberFloatType = double, - template class AllocatorType = std::allocator, - template class JSONSerializer = - adl_serializer, - class BinaryType = std::vector> -class basic_json; - -/*! -@brief JSON Pointer - -A JSON pointer defines a string syntax for identifying a specific value -within a JSON document. It can be used with functions `at` and -`operator[]`. Furthermore, JSON pointers are the base for JSON patches. - -@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) - -@since version 2.0.0 -*/ -template -class json_pointer; - -/*! -@brief default JSON class - -This type is the default specialization of the @ref basic_json class which -uses the standard template types. - -@since version 1.0.0 -*/ -using json = basic_json<>; - -template -struct ordered_map; - -/*! -@brief ordered JSON class - -This type preserves the insertion order of object keys. - -@since version 3.9.0 -*/ -using ordered_json = basic_json; - -} // namespace nlohmann - -#endif // INCLUDE_NLOHMANN_JSON_FWD_HPP_ diff --git a/builtins/xrootd/CMakeLists.txt b/builtins/xrootd/CMakeLists.txt new file mode 100644 index 0000000000000..4e0b7104af545 --- /dev/null +++ b/builtins/xrootd/CMakeLists.txt @@ -0,0 +1,93 @@ +# Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. +# All rights reserved. +# +# For the licensing terms see $ROOTSYS/LICENSE. +# For the list of contributors see $ROOTSYS/README/CREDITS. + +include(ExternalProject) + +find_package(OpenSSL REQUIRED) + +set(XROOTD_VERSION "5.3.1") +set(XROOTD_VERSIONNUM 500030001 CACHE INTERNAL "" FORCE) +set(XROOTD_SRC_URI https://github.com/xrootd/xrootd/archive/v${XROOTD_VERSION}.tar.gz) +set(XROOTD_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/XROOTD-prefix) +message(STATUS "Downloading and building XROOTD version ${XROOTD_VERSION}") + +# Guess under which directory XRootD will install its libraires +set(XROOTD_LIBDIR "lib") +if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND ${CMAKE_SIZEOF_VOID_P} EQUAL 8 +AND NOT CMAKE_CROSSCOMPILING AND NOT EXISTS "/etc/debian_version") + set(XROOTD_LIBDIR "lib64") +endif() + +foreach(lib XrdUtils XrdCl) + set(libname ${CMAKE_SHARED_LIBRARY_PREFIX}${lib}${CMAKE_SHARED_LIBRARY_SUFFIX}) + list(APPEND XROOTD_LIBRARIES ${XROOTD_PREFIX}/${XROOTD_LIBDIR}/${libname}) +endforeach() + +ExternalProject_Add( + XROOTD + URL ${XROOTD_SRC_URI} + URL_HASH SHA256=0800b4c6444591a6514c5b81b2c4c79439875136ab2267ce4bed99cb94ead6e6 + INSTALL_DIR ${XROOTD_PREFIX} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= + -DCMAKE_PREFIX_PATH:STRING=${OPENSSL_PREFIX} + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_CXX_FLAGS=${ROOT_EXTERNAL_CXX_FLAGS} + -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} + -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} + -DENABLE_PYTHON=OFF + -DENABLE_CEPH=OFF + -DCMAKE_INSTALL_RPATH:STRING=${XROOTD_PREFIX}/${XROOTD_LIBDIR} + INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install + COMMAND ${CMAKE_COMMAND} -E copy_directory /include/xrootd /include + LOG_DOWNLOAD 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 + BUILD_BYPRODUCTS ${XROOTD_LIBRARIES} + TIMEOUT 600 +) + +if(builtin_openssl) + add_dependencies(XROOTD OPENSSL) +endif() + +list(APPEND XROOTD_LIBRARIES OpenSSL::SSL) + +unset(XROOTD_FOUND CACHE) +unset(XROOTD_FOUND PARENT_SCOPE) +set(XROOTD_FOUND TRUE CACHE BOOL "" FORCE) + +set(XROOTD_INCLUDE_DIRS ${XROOTD_PREFIX}/include/xrootd ${XROOTD_DESTDIR}/include/xrootd/private) +set(XROOTD_NOMAIN TRUE) +set(XROOTD_CFLAGS "-DROOTXRDVERS=${XROOTD_VERSIONNUM}") + +set(XROOTD_INCLUDE_DIR ${XROOTD_PREFIX}/include/xrootd ${XROOTD_PREFIX}/include/xrootd/private CACHE INTERNAL "" FORCE) +set(XROOTD_INCLUDE_DIRS ${XROOTD_PREFIX}/include/xrootd ${XROOTD_PREFIX}/include/xrootd/private CACHE INTERNAL "" FORCE) +set(XROOTD_LIBRARY ${XROOTD_PREFIX}/${XROOTD_LIBDIR}/${XROOTD_LIBNAME} CACHE INTERNAL "" FORCE) +set(XROOTD_LIBRARIES ${XROOTD_LIBRARIES} CACHE INTERNAL "" FORCE) + +add_library(xrootd INTERFACE) +target_include_directories(xrootd INTERFACE $) +target_link_libraries(xrootd INTERFACE $) +set_property(TARGET xrootd PROPERTY INTERFACE_BUILD_RPATH ${XROOTD_PREFIX}/${XROOTD_LIBDIR}) +add_dependencies(xrootd XROOTD) + +add_library(Xrootd::Xrootd ALIAS xrootd) + +set_property(GLOBAL APPEND PROPERTY ROOT_BUILTIN_TARGETS XROOTD) + +install(DIRECTORY ${XROOTD_PREFIX}/${XROOTD_LIBDIR}/ DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries FILES_MATCHING PATTERN "libXrd*") +install(DIRECTORY ${XROOTD_PREFIX}/include/xrootd/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT headers) +if(APPLE) + # XRootD libraries on mac need the LC_RPATH variable set. The build process already takes care of setting + # * BUILD_RPATH = build/XROOTD-prefix/../src + # * INSTALL_RPATH = build/lib + # Since the install directory for the builtin_xrootd target corresponds to the build directory of the main project. + # Use a post install script to change the LC_RPATH variable of the libraries in the ROOT install folder. + install(SCRIPT ${CMAKE_CURRENT_LIST_DIR}/XROOTDApplePostInstall.cmake + CODE "xrootd_libs_change_rpath(${XROOTD_PREFIX}/${XROOTD_LIBDIR} ${CMAKE_INSTALL_FULL_LIBDIR})" + ) +endif() diff --git a/cmake/modules/XROOTDApplePostInstall.cmake b/builtins/xrootd/XROOTDApplePostInstall.cmake similarity index 100% rename from cmake/modules/XROOTDApplePostInstall.cmake rename to builtins/xrootd/XROOTDApplePostInstall.cmake diff --git a/cmake/modules/CheckCompiler.cmake b/cmake/modules/CheckCompiler.cmake index 437d84b47077c..6d7b324ec1430 100644 --- a/cmake/modules/CheckCompiler.cmake +++ b/cmake/modules/CheckCompiler.cmake @@ -115,7 +115,7 @@ include(CheckCCompilerFlag) # so we check the value of __cplusplus. # This default value can be overridden by specifying one at the prompt. if (MSVC) - set(CXX_STANDARD_STRING 2011) + set(CXX_STANDARD_STRING "201703L") else() execute_process(COMMAND echo __cplusplus COMMAND ${CMAKE_CXX_COMPILER} -E -x c++ - diff --git a/cmake/modules/FindDask.cmake b/cmake/modules/FindDask.cmake new file mode 100644 index 0000000000000..ec12efa008937 --- /dev/null +++ b/cmake/modules/FindDask.cmake @@ -0,0 +1,43 @@ +# Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. +# All rights reserved. +# +# For the licensing terms see $ROOTSYS/LICENSE. +# For the list of contributors see $ROOTSYS/README/CREDITS. + +# Find if dask is installed on the environment of the main Python executable +# used to build ROOT. +# +# Installing the package `dask` through conda always provides the `distributed` +# The same is not true for pip, where the two packages have to be installed +# separately. +# +# This module sets the following variables +# Dask_FOUND - system has dask and it is usable +# Dask_DEPENDENCIES_READY - the environment could import the `dask` and `distributed` packages +# Dask_VERSION_STRING - Dask version string + +# Import `dask` and `distributed` using the main Python executable, print dask version +execute_process( + COMMAND ${PYTHON_EXECUTABLE_Development_Main} -c "import distributed; import dask; print(dask.__version__)" + RESULT_VARIABLE _DASK_IMPORT_EXIT_STATUS + OUTPUT_VARIABLE _DASK_VALUES_OUTPUT + ERROR_VARIABLE _DASK_ERROR_VALUE + OUTPUT_STRIP_TRAILING_WHITESPACE +) + +# Exit status equal to zero means success +if(_DASK_IMPORT_EXIT_STATUS EQUAL 0) + # Build the version string + string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" Dask_VERSION_STRING "${_DASK_VALUES_OUTPUT}") + # Signal to CMake that the environment could import `dask` and `distributed` packages + set(Dask_DEPENDENCIES_READY TRUE) +else() + message(STATUS "Python package 'dask' could not be imported with ${PYTHON_EXECUTABLE_Development_Main}\n" + "${_DASK_ERROR_VALUE}" + ) +endif() + +find_package_handle_standard_args(Dask + REQUIRED_VARS Dask_DEPENDENCIES_READY + VERSION_VAR Dask_VERSION_STRING +) diff --git a/cmake/modules/FindXROOTD.cmake b/cmake/modules/FindXROOTD.cmake index 838c41c3c03d5..fe48c0162a8d8 100644 --- a/cmake/modules/FindXROOTD.cmake +++ b/cmake/modules/FindXROOTD.cmake @@ -169,6 +169,12 @@ if(XROOTD_FOUND) endif () endif() +if(XROOTD_FOUND AND NOT TARGET Xrootd::Xrootd) + add_library(Xrootd::Xrootd INTERFACE IMPORTED) + set_property(TARGET Xrootd::Xrootd PROPERTY INTERFACE_INCLUDE_DIRECTORIES "${XROOTD_INCLUDE_DIRS}") + set_property(TARGET Xrootd::Xrootd PROPERTY INTERFACE_LINK_LIBRARIES "${XROOTD_LIBRARIES}") +endif() + mark_as_advanced(XROOTD_INCLUDE_DIR XROOTD_XrdMain_LIBRARY XROOTD_XrdUtils_LIBRARY diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index d90367761460d..7c864b93289a6 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -121,6 +121,7 @@ ROOT_BUILD_OPTION(cxxmodules OFF "Enable support for C++ modules") ROOT_BUILD_OPTION(daos OFF "Enable RNTuple support for Intel DAOS") ROOT_BUILD_OPTION(dataframe ON "Enable ROOT RDataFrame") ROOT_BUILD_OPTION(test_distrdf_pyspark OFF "Enable distributed RDataFrame tests that use pyspark") +ROOT_BUILD_OPTION(test_distrdf_dask OFF "Enable distributed RDataFrame tests that use dask") ROOT_BUILD_OPTION(davix ON "Enable support for Davix (HTTP/WebDAV access)") ROOT_BUILD_OPTION(dcache OFF "Enable support for dCache (requires libdcap from DESY)") ROOT_BUILD_OPTION(dev OFF "Enable recommended developer compilation flags, reduce exposed includes") @@ -155,7 +156,8 @@ ROOT_BUILD_OPTION(pyroot_legacy OFF "Use legacy Python bindings for ROOT") ROOT_BUILD_OPTION(pythia6_nolink OFF "Delayed linking of Pythia6 library") ROOT_BUILD_OPTION(pythia6 ON "Enable support for Pythia 6.x") ROOT_BUILD_OPTION(pythia8 ON "Enable support for Pythia 8.x") -ROOT_BUILD_OPTION(qt5web OFF "Enable support for Qt5 web-based display (requires Qt5WebEngine)") +ROOT_BUILD_OPTION(qt5web OFF "Enable support for Qt5 web-based display (requires Qt5::WebEngine and Qt5::WebEngineWidgets)") +ROOT_BUILD_OPTION(qt6web OFF "Enable support for Qt6 web-based display (requires Qt6::WebEngineCore and Qt6::WebEngineWidgets)") ROOT_BUILD_OPTION(r OFF "Enable support for R bindings (requires R, Rcpp, and RInside)") ROOT_BUILD_OPTION(roofit ON "Build the advanced fitting package RooFit, and RooStats for statistical tests. If xml is available, also build HistFactory.") ROOT_BUILD_OPTION(webgui ON "Build Web-based UI components of ROOT (requires C++17 standard or higher)") @@ -224,6 +226,7 @@ if(all) set(cuda_defvalue ON) set(dataframe_defvalue ON) set(test_distrdf_pyspark_defvalue ON) + set(test_distrdf_dask_defvalue ON) set(davix_defvalue ON) set(dcache_defvalue ON) set(fftw3_defvalue ON) @@ -399,7 +402,6 @@ if(webgui) endif() endif() - #---Removed options------------------------------------------------------------ foreach(opt afdsmgrd afs bonjour castor chirp geocad glite globus hdfs ios krb5 ldap memstat qt qtgsi rfio ruby sapdb srp table python vmc) @@ -484,11 +486,20 @@ elseif(Python3_Interpreter_Development_FOUND AND Python2_Interpreter_Development endif() #---distributed RDataFrame pyspark tests require both dataframe and pyroot---------------------------------- -if(test_distrdf_pyspark AND (NOT dataframe OR NOT pyroot)) - - message(STATUS "Running the tests for distributed RDataFrame with pyspark requires both RDataFrame and PyROOT to be enabled") - message(STATUS " Switching it OFF because either pyroot or dataframe option is disabled") - message(STATUS " pyroot is set to ${pyroot} and dataframe is set to ${dataframe}") - set(test_distrdf_pyspark OFF CACHE BOOL "Disabled because either dataframe or pyroot were disabled" FORCE) +if(test_distrdf_pyspark OR test_distrdf_dask) + + if(NOT dataframe OR NOT pyroot) + message(STATUS "Running the tests for distributed RDataFrame requires both RDataFrame and PyROOT to be enabled") + message(STATUS " Switching OFF the tests because either pyroot or dataframe option is disabled") + message(STATUS " pyroot is set to ${pyroot} and dataframe is set to ${dataframe}") + set(test_distrdf_pyspark OFF CACHE BOOL "Disabled because either dataframe or pyroot were disabled" FORCE) + set(test_distrdf_dask OFF CACHE BOOL "Disabled because either dataframe or pyroot were disabled" FORCE) + elseif(NOT PYTHON_VERSION_STRING_Development_Main VERSION_GREATER_EQUAL 3.7) + message(STATUS "Distributed RDataFrame requires Python 3.7 and above.") + message(STATUS " The current Python version is ${PYTHON_VERSION_STRING_Development_Main}.") + message(STATUS " Setting options 'test_distrdf_*' to OFF.") + set(test_distrdf_pyspark OFF CACHE BOOL "Disabled because Python version is less than minimum required 3.7" FORCE) + set(test_distrdf_dask OFF CACHE BOOL "Disabled because Python version is less than minimum required 3.7" FORCE) + endif() endif() diff --git a/cmake/modules/RootMacros.cmake b/cmake/modules/RootMacros.cmake index c536c5999b6ae..9651a3dbbeb1a 100644 --- a/cmake/modules/RootMacros.cmake +++ b/cmake/modules/RootMacros.cmake @@ -286,8 +286,8 @@ function(ROOT_GENERATE_DICTIONARY dictionary) set(libprefix "") endif() - # list of include directories for dictionary generation - set(incdirs) + # list of include directories for dictionary generation + set(incdirs) if((CMAKE_PROJECT_NAME STREQUAL ROOT) AND (TARGET ${ARG_MODULE})) set(headerdirs) @@ -609,11 +609,10 @@ function(ROOT_GENERATE_DICTIONARY dictionary) # get target properties added after call to ROOT_GENERATE_DICTIONARY() if(TARGET ${ARG_MODULE}) - if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.15) - set(module_incs $>) - else() - set(module_incs $) - endif() + # NOTE that module_sysincs is already part of ${module_sysincs}. But -isystem "wins", + # and list exclusion for generator expressions is too complex. + set(module_incs $>) + set(module_sysincs $>) set(module_defs $) endif() endif() @@ -627,11 +626,18 @@ function(ROOT_GENERATE_DICTIONARY dictionary) endforeach() endif() + set(compIncPaths) + foreach(implinc IN LISTS CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES CMAKE_C_IMPLICIT_INCLUDE_DIRECTORIES) + list(APPEND compIncPaths "-compilerI${implinc}") + endforeach() + #---call rootcint------------------------------------------ add_custom_command(OUTPUT ${dictionary}.cxx ${pcm_name} ${rootmap_name} ${cpp_module_file} COMMAND ${command} -v2 -f ${dictionary}.cxx ${newargs} ${excludepathsargs} ${rootmapargs} ${ARG_OPTIONS} ${definitions} "$<$:-D$>" + ${compIncPaths} + "$<$:-isystem;$>" ${includedirs} "$<$:-I$>" ${headerfiles} ${_linkdef} IMPLICIT_DEPENDS ${_implicitdeps} @@ -914,8 +920,10 @@ function(ROOT_LINKER_LIBRARY library) get_target_property(_target_type ${lib} TYPE) if(${_target_type} STREQUAL "INTERFACE_LIBRARY") get_target_property(lib_incdirs ${lib} INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(lib_rpath ${lib} INTERFACE_BUILD_RPATH) else() get_target_property(lib_incdirs ${lib} INCLUDE_DIRECTORIES) + get_target_property(lib_rpath ${lib} BUILD_RPATH) endif() if(lib_incdirs) foreach(dir ${lib_incdirs}) @@ -923,6 +931,9 @@ function(ROOT_LINKER_LIBRARY library) list(APPEND dep_inc_list ${dir}) endforeach() endif() + if(lib_rpath) + set_target_properties(${library} PROPERTIES BUILD_RPATH ${lib_rpath}) + endif() endif() endforeach() endif() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 863289b12f2e2..19c3339390f37 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -74,8 +74,8 @@ if(NOT builtin_nlohmannjson) else() find_package(nlohmann_json QUIET) if(nlohmann_json_FOUND) - get_target_property(_nlohmann_json_inlc nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES) - message(STATUS "Found nlohmann/json.hpp in ${_nlohmann_json_inlc} (found version ${nlohmann_json_VERSION})") + get_target_property(_nlohmann_json_incl nlohmann_json::nlohmann_json INTERFACE_INCLUDE_DIRECTORIES) + message(STATUS "Found nlohmann/json.hpp in ${_nlohmann_json_incl} (found version ${nlohmann_json_VERSION})") else() message(STATUS "nlohmann/json.hpp not found. Switching on builtin_nlohmannjson option") set(builtin_nlohmannjson ON CACHE BOOL "Enabled because nlohmann/json.hpp not found" FORCE) @@ -954,13 +954,18 @@ if(monalisa) endif() #---Check for Xrootd support--------------------------------------------------------- + +foreach(suffix FOUND INCLUDE_DIR INCLUDE_DIRS LIBRARY LIBRARIES) + unset(XROOTD_${suffix} CACHE) +endforeach() + if(xrootd AND NOT builtin_xrootd) message(STATUS "Looking for XROOTD") find_package(XROOTD) if(NOT XROOTD_FOUND) if(fail-on-missing) message(FATAL_ERROR "XROOTD not found. Set environment variable XRDSYS to point to your XROOTD installation, " - "or inlcude the installation of XROOTD in the CMAKE_PREFIX_PATH. " + "or include the installation of XROOTD in the CMAKE_PREFIX_PATH. " "Alternatively, you can also enable the option 'builtin_xrootd' to build XROOTD internally") else() message(STATUS "XROOTD not found, enabling 'builtin_xrootd' option") @@ -981,65 +986,23 @@ if(builtin_xrootd AND NO_CONNECTION) endif() endif() if(builtin_xrootd) - set(XROOTD_VERSION 4.12.8) - set(XROOTD_VERSIONNUM 400120008) - set(XROOTD_SRC_URI ${lcgpackages}/xrootd-${XROOTD_VERSION}.tar.gz) - set(XROOTD_DESTDIR ${CMAKE_BINARY_DIR}) - set(XROOTD_ROOTDIR ${XROOTD_DESTDIR}) - message(STATUS "Downloading and building XROOTD version ${xrootd_version}") - - # Guess under which directory XRootD will install its libraires - set(XROOTD_LIBDIR "lib") - if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND ${CMAKE_SIZEOF_VOID_P} EQUAL 8 - AND NOT CMAKE_CROSSCOMPILING AND NOT EXISTS "/etc/debian_version") - set(XROOTD_LIBDIR "lib64") - endif() - - set(XROOTD_LIBRARIES ${XROOTD_ROOTDIR}/${XROOTD_LIBDIR}/libXrdUtils${CMAKE_SHARED_LIBRARY_SUFFIX} - ${XROOTD_ROOTDIR}/${XROOTD_LIBDIR}/libXrdClient${CMAKE_SHARED_LIBRARY_SUFFIX} - ${XROOTD_ROOTDIR}/${XROOTD_LIBDIR}/libXrdCl${CMAKE_SHARED_LIBRARY_SUFFIX}) - ExternalProject_Add( - XROOTD - URL ${XROOTD_SRC_URI} - URL_HASH SHA256=86d8e4bd7382fb3053002cf3d58b997623d1d26db93c8891080603827f01b4cd - INSTALL_DIR ${XROOTD_ROOTDIR} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH= - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_CXX_FLAGS=${ROOT_EXTERNAL_CXX_FLAGS} - -DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT} - -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET} - -DENABLE_PYTHON=OFF - -DENABLE_CEPH=OFF - -DCMAKE_INSTALL_RPATH:STRING=${XROOTD_ROOTDIR}/${XROOTD_LIBDIR} - INSTALL_COMMAND ${CMAKE_COMMAND} --build . --target install - COMMAND ${CMAKE_COMMAND} -E copy_directory /include/xrootd /include - LOG_DOWNLOAD 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 - BUILD_BYPRODUCTS ${XROOTD_LIBRARIES} - TIMEOUT 600 - ) - # We cannot call find_package(XROOTD) becuase the package is not yet built. So, we need to emulate what it defines.... + list(APPEND ROOT_BUILTINS XROOTD) + add_subdirectory(builtins/xrootd) + set(xrootd ON CACHE BOOL "Enabled because builtin_xrootd requested (${xrootd_description})" FORCE) +endif() - set(XROOTD_INCLUDE_DIRS ${XROOTD_ROOTDIR}/include/xrootd ${XROOTD_ROOTDIR}/include/xrootd/private) - set(XROOTD_NOMAIN TRUE) - set(XROOTD_CFLAGS "-DROOTXRDVERS=${XROOTD_VERSIONNUM}") - install(DIRECTORY ${XROOTD_ROOTDIR}/${XROOTD_LIBDIR}/ DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT libraries FILES_MATCHING PATTERN "libXrd*") - install(DIRECTORY ${XROOTD_ROOTDIR}/include/xrootd/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT headers) - if(APPLE) - # XRootD libraries on mac need the LC_RPATH variable set. The build process already takes care of setting - # * BUILD_RPATH = build/XROOTD-prefix/../src - # * INSTALL_RPATH = build/lib - # Since the install directory for the builtin_xrootd target corresponds to the build directory of the main project. - # Use a post install script to change the LC_RPATH variable of the libraries in the ROOT install folder. - install(SCRIPT ${CMAKE_CURRENT_LIST_DIR}/XROOTDApplePostInstall.cmake - CODE "xrootd_libs_change_rpath(${XROOTD_ROOTDIR}/${XROOTD_LIBDIR} ${CMAKE_INSTALL_FULL_LIBDIR})" - ) +if(xrootd AND XROOTD_VERSIONNUM VERSION_GREATER_EQUAL 500000000) + if(xproofd) + if(fail-on-missing) + message(FATAL_ERROR "XROOTD is version 5 or greater. The legacy xproofd servers can not be built with this version. Use -Dxproofd:BOOL=OFF to disable.") + else() + message(STATUS "XROOTD is version 5 or greater. The legacy xproofd servers can not be built with this version. Disabling 'xproofd' option.") + set(xproofd OFF CACHE BOOL "Disabled because xrootd version is 5 or greater" FORCE) + endif() endif() - set(XROOTD_TARGET XROOTD) - set(xrootd ON CACHE BOOL "Enabled because builtin_xrootd requested (${xrootd_description})" FORCE) endif() + +#---check if netxng and netx can be built------------------------------- if(xrootd AND XROOTD_VERSIONNUM VERSION_GREATER 300030005) set(netxng ON) else() @@ -1050,16 +1013,6 @@ if(xrootd AND XROOTD_VERSIONNUM VERSION_LESS 500000000) else() set(netx OFF) endif() -if(xrootd AND XROOTD_VERSIONNUM VERSION_GREATER_EQUAL 500000000) - if(xproofd) - if(fail-on-missing) - message(FATAL_ERROR "XROOTD is version 5 or greater. The legacy xproofd servers can not be built with this version. Use -Dxproofd:BOOL=OFF to disable.") - else() - message(STATUS "XROOTD is version 5 or greater. The legacy xproofd servers can not be built with this version. Disabling 'xproofd' option.") - set(xproofd OFF CACHE BOOL "Disabled because xrootd version is 5 or greater" FORCE) - endif() - endif() -endif() #---Alien support---------------------------------------------------------------- if(alien) @@ -1365,7 +1318,7 @@ if(builtin_tbb) URL_HASH SHA256=${tbb_sha256} INSTALL_DIR ${CMAKE_BINARY_DIR} CONFIGURE_COMMAND devenv.exe /useenv /upgrade build/${vsdir}/makefile.sln - BUILD_COMMAND devenv.exe /useenv /build "Release|${tbb_arch}" build/${vsdir}/makefile.sln + BUILD_COMMAND MSBuild.exe build/${vsdir}/makefile.sln /p:Configuration=Release /p:Platform=${tbb_arch} INSTALL_COMMAND ${CMAKE_COMMAND} -E copy_if_different build/${vsdir}/${tbb_arch}/Release/tbb.dll ${CMAKE_BINARY_DIR}/bin/ COMMAND ${CMAKE_COMMAND} -E copy_if_different build/${vsdir}/${tbb_arch}/Release/tbbmalloc.dll ${CMAKE_BINARY_DIR}/bin/ COMMAND ${CMAKE_COMMAND} -E copy_if_different build/${vsdir}/${tbb_arch}/Release/tbbmalloc_proxy.dll ${CMAKE_BINARY_DIR}/bin/ @@ -1942,7 +1895,7 @@ if (testing) googletest GIT_REPOSITORY https://github.com/google/googletest.git GIT_SHALLOW 1 - GIT_TAG release-1.10.0 + GIT_TAG release-1.11.0 UPDATE_COMMAND "" # TIMEOUT 10 # # Force separate output paths for debug and release builds to allow easy @@ -2108,3 +2061,23 @@ if(test_distrdf_pyspark) endif() endif() + +#------------------------------------------------------------------------------------ +# Check if the dask package is installed on the system. +# Needed to run tests of the distributed RDataFrame module that use dask. +if(test_distrdf_dask) + message(STATUS "Looking for Dask") + + if(fail-on-missing) + find_package(Dask 2020.12 REQUIRED) + else() + + find_package(Dask 2020.12) + if(NOT Dask_FOUND) + message(STATUS "Switching OFF 'test_distrdf_dask' option") + set(test_distrdf_dask OFF CACHE BOOL "Disabled because Dask not found" FORCE) + endif() + + endif() + +endif() diff --git a/cmake/modules/SetUpWindows.cmake b/cmake/modules/SetUpWindows.cmake index 42fd22ee0136d..846ca8784f20c 100644 --- a/cmake/modules/SetUpWindows.cmake +++ b/cmake/modules/SetUpWindows.cmake @@ -19,7 +19,7 @@ if(CMAKE_COMPILER_IS_GNUCXX) elseif(MSVC) if("${CMAKE_GENERATOR_PLATFORM}" MATCHES "x64") - set(ARCH -D_WIN64) + set(ARCH "-wd4267 -D_WIN64") set(ROOT_ARCHITECTURE win64) else() set(ROOT_ARCHITECTURE win32) @@ -39,6 +39,9 @@ elseif(MSVC) if(CMAKE_PROJECT_NAME STREQUAL ROOT) set(CMAKE_CXX_FLAGS "-nologo -I${CMAKE_SOURCE_DIR}/build/win -FIw32pragma.h -FIsehmap.h ${BLDCXXFLAGS} -EHsc- -W3 -wd4141 -wd4291 -wd4244 -wd4049 -wd4146 -D_WIN32 ${ARCH} -D_XKEYCHECK_H -DNOMINMAX -D_CRT_SECURE_NO_WARNINGS") set(CMAKE_C_FLAGS "-nologo -I${CMAKE_SOURCE_DIR}/build/win -FIw32pragma.h -FIsehmap.h ${BLDCFLAGS} -EHsc- -W3 -D_WIN32 ${ARCH} -DNOMINMAX") + if(CMAKE_CXX_STANDARD GREATER_EQUAL 17) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_SILENCE_CXX17_ITERATOR_BASE_CLASS_DEPRECATION_WARNING -D_SILENCE_CXX17_CODECVT_HEADER_DEPRECATION_WARNING") + endif() if(win_broken_tests) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DR__ENABLE_BROKEN_WIN_TESTS") endif() diff --git a/config/root-config.in b/config/root-config.in index 77f913b2a4772..a1efa1c80e72c 100755 --- a/config/root-config.in +++ b/config/root-config.in @@ -347,7 +347,7 @@ win32) ;; win32gcc | win64gcc) # Windows (32 or 64) with cygwin gcc - auxcflags="-std=gnu++11" + auxcflags="-std=gnu++14" auxlibs= forcelibs=$rootulibs forceglibs=$rootuglibs diff --git a/core/base/inc/TDirectory.h b/core/base/inc/TDirectory.h index f8d7f6b446c19..fd4326ed51ee8 100644 --- a/core/base/inc/TDirectory.h +++ b/core/base/inc/TDirectory.h @@ -255,17 +255,38 @@ can be replaced with the simpler and exception safe: virtual Int_t Write(const char * /*name*/=nullptr, Int_t /*opt*/=0, Int_t /*bufsize*/=0) const override {return 0;} virtual Int_t WriteTObject(const TObject *obj, const char *name =nullptr, Option_t * /*option*/="", Int_t /*bufsize*/ =0); private: +/// \cond HIDDEN_SYMBOLS Int_t WriteObject(void *obj, const char* name, Option_t *option="", Int_t bufsize=0); // Intentionally not implemented. +/// \endcond public: - /// Write an object with proper type checking. + /// \brief Write an object with proper type checking. /// \param[in] obj Pointer to an object to be written. /// \param[in] name Name of the object in the file. - /// \param[in] option Options. See TDirectory::WriteTObject() or TDirectoryWriteObjectAny(). - /// \param[in] bufsize Buffer size. See TDirectory::WriteTObject(). - template inline Int_t WriteObject(const T* obj, const char* name, Option_t *option="", Int_t bufsize=0) - { - return WriteObjectAny(obj, TClass::GetClass(), name, option, bufsize); - } + /// \param[in] option Options. See TDirectory::WriteTObject. + /// \param[in] bufsize Buffer size. See TDirectory::WriteTObject. + /// + /// This overload takes care of instances of classes that are not derived + /// from TObject. The method redirects to TDirectory::WriteObjectAny. + template + inline std::enable_if_t::value, Int_t> + WriteObject(const T *obj, const char *name, Option_t *option = "", Int_t bufsize = 0) + { + return WriteObjectAny(obj, TClass::GetClass(), name, option, bufsize); + } + /// \brief Write an object with proper type checking. + /// \param[in] obj Pointer to an object to be written. + /// \param[in] name Name of the object in the file. + /// \param[in] option Options. See TDirectory::WriteTObject. + /// \param[in] bufsize Buffer size. See TDirectory::WriteTObject. + /// + /// This overload takes care of instances of classes that are derived from + /// TObject. The method redirects to TDirectory::WriteTObject. + template + inline std::enable_if_t::value, Int_t> + WriteObject(const T *obj, const char *name, Option_t *option = "", Int_t bufsize = 0) + { + return WriteTObject(obj, name, option, bufsize); + } virtual Int_t WriteObjectAny(const void *, const char * /*classname*/, const char * /*name*/, Option_t * /*option*/="", Int_t /*bufsize*/ =0) {return 0;} virtual Int_t WriteObjectAny(const void *, const TClass * /*cl*/, const char * /*name*/, Option_t * /*option*/="", Int_t /*bufsize*/ =0) {return 0;} virtual void WriteDirHeader() {} diff --git a/core/base/inc/TMathBase.h b/core/base/inc/TMathBase.h index 6264679254d0a..8df6f0317380b 100644 --- a/core/base/inc/TMathBase.h +++ b/core/base/inc/TMathBase.h @@ -127,11 +127,7 @@ inline Long_t TMath::Abs(Long_t d) { return std::labs(d); } inline Long64_t TMath::Abs(Long64_t d) -#if __cplusplus >= 201103 { return std::llabs(d); } -#else -{ return (d >= 0) ? d : -d; } -#endif inline Float_t TMath::Abs(Float_t d) { return std::abs(d); } diff --git a/core/base/inc/TRemoteObject.h b/core/base/inc/TRemoteObject.h index 48b198dc9dc3c..64c1d882c1ae1 100644 --- a/core/base/inc/TRemoteObject.h +++ b/core/base/inc/TRemoteObject.h @@ -57,7 +57,7 @@ class TRemoteObject : public TNamed { void SetFolder(Bool_t isFolder) { fIsFolder = isFolder; } void SetKeyObjectName(const char *name) { fKeyObjectName = name; } void SetKeyClassName(const char *name) { fKeyClassName = name; } - void SetRemoteAddress(Long_t addr) { fRemoteAddress = addr; } + void SetRemoteAddress(Longptr_t addr) { fRemoteAddress = addr; } ClassDef(TRemoteObject,0) //A remote object }; diff --git a/core/base/inc/Varargs.h b/core/base/inc/Varargs.h index 525f1f64a688e..30983e01240f9 100644 --- a/core/base/inc/Varargs.h +++ b/core/base/inc/Varargs.h @@ -44,12 +44,10 @@ #endif #if !defined(R__VA_COPY) -# if defined(__GNUC__) && !defined(__FreeBSD__) -# define R__VA_COPY(to, from) __va_copy((to), (from)) +# if defined(va_copy) +# define R__VA_COPY(to, from) va_copy((to), (from)) # elif defined(__va_copy) # define R__VA_COPY(to, from) __va_copy((to), (from)) -# elif defined(va_copy) -# define R__VA_COPY(to, from) va_copy((to), (from)) # elif defined(_WIN32) && _MSC_VER < 1310 # define R__VA_COPY(to, from) (*(to) = *(from)) # else diff --git a/core/base/src/TApplication.cxx b/core/base/src/TApplication.cxx index 3637d3acfbfb7..820373a643d26 100644 --- a/core/base/src/TApplication.cxx +++ b/core/base/src/TApplication.cxx @@ -401,7 +401,10 @@ void TApplication::GetOptions(Int_t *argc, char **argv) TString argw; if (gROOT->IsBatch()) argw = "batch"; if (*opt == '=') argw.Append(opt+1); - if (gSystem->Load("libROOTWebDisplay") >= 0) { + if (argw == "off") { + gROOT->SetWebDisplay(argw.Data()); + gEnv->SetValue("Browser.Name", "TRootBrowser"); // force usage of TBrowser back + } else if (gSystem->Load("libROOTWebDisplay") >= 0) { gROOT->SetWebDisplay(argw.Data()); gEnv->SetValue("Gui.Factory", "web"); } else { diff --git a/core/base/src/TAttAxis.cxx b/core/base/src/TAttAxis.cxx index db16ad330f4b0..f313c7327c3f1 100644 --- a/core/base/src/TAttAxis.cxx +++ b/core/base/src/TAttAxis.cxx @@ -28,17 +28,18 @@ Manages histogram axis attributes. They are: - - The number of divisions - - The line axis' color - - The labels' color - - The labels' font - - The labels' offset - - The labels' size - - The tick marks' - - The axis title's offset - - The axis title's size - - The axis title's color - - The axis title's font + - The number of divisions: TAttAxis::SetNdivisions. + - The line axis' color: TAttAxis::SetAxisColor. + - The axis labels' color: TAttAxis::SetLabelColor. + - The axis labels' font: TAttAxis::SetLabelFont. + - The axis labels' offset: TAttAxis::SetLabelOffset. + - The axis labels' size: TAttAxis::SetLabelSize. + - The tick marks's length: TAttAxis::SetTickLength or TAttAxis::SetTickSize . + - The axis title's offset: TAttAxis::SetTitleOffset. + - The axis title's size: TAttAxis::SetTitleSize. + - The axis title's color: TAttAxis::SetTitleColor. + - The axis title's font: TAttAxis::SetTitleFont. + */ TAttAxis::TAttAxis() @@ -188,6 +189,7 @@ void TAttAxis::SetLabelFont(Style_t font) //////////////////////////////////////////////////////////////////////////////// /// Set distance between the axis and the labels. /// The distance is expressed in per cent of the pad width. +/// A negative value allow to draw the label on the other side of the axis. void TAttAxis::SetLabelOffset(Float_t offset) { @@ -220,10 +222,17 @@ void TAttAxis::SetLabelSize(Float_t size) /// n2 is the number of second order divisions and /// n3 is the number of third order divisions. /// -/// e.g. 512 means 12 primary and 5 secondary divisions. -/// /// If the number of divisions is "optimized" (see above) n1, n2, n3 are /// maximum values. +/// +/// Examples: +/// +/// - ndiv = 0: no tick marks. +/// - ndiv = 2: 2 divisions, one tick mark in the middle of the axis. +/// - ndiv = 510: 10 primary divisions, 5 secondary divisions. +/// - ndiv = -10: exactly 10 primary divisions. + + void TAttAxis::SetNdivisions(Int_t n, Bool_t optim) { diff --git a/core/base/src/TAttMarker.cxx b/core/base/src/TAttMarker.cxx index 481cb6c63e467..17a129a2b0e8f 100644 --- a/core/base/src/TAttMarker.cxx +++ b/core/base/src/TAttMarker.cxx @@ -134,6 +134,8 @@ Begin_Macro } End_Macro +\warning Non-symmetric symbols should be used carefully. See markerwarning.C + \anchor ATTMARKER21 ### Marker line width diff --git a/core/base/src/TDirectory.cxx b/core/base/src/TDirectory.cxx index 3fed536b1f0fc..98a2316565fe2 100644 --- a/core/base/src/TDirectory.cxx +++ b/core/base/src/TDirectory.cxx @@ -1306,23 +1306,16 @@ void TDirectory::RegisterContext(TContext *ctxt) { } //////////////////////////////////////////////////////////////////////////////// -/// Register a std::atomic pointing to this TDirectory object +/// Register a std::atomic that will soon be pointing to this TDirectory object void TDirectory::RegisterGDirectory(std::atomic *globalptr) { ROOT::Internal::TSpinLockGuard slg(fSpinLock); - auto oldvalue = globalptr->load(); - - auto iter = std::find(fGDirectories.begin(), fGDirectories.end(), globalptr); - if (iter != fGDirectories.begin()) + if (std::find(fGDirectories.begin(), fGDirectories.end(), globalptr) != fGDirectories.end()) fGDirectories.push_back(globalptr); - - if (oldvalue && oldvalue != this) { - iter = std::find(oldvalue->fGDirectories.begin(), oldvalue->fGDirectories.end(), globalptr); - if (iter != oldvalue->fGDirectories.begin()) - oldvalue->fGDirectories.erase(iter); - } + // globalptr->load()->fGDirectories will still contain globalptr, but we cannot + // know whether globalptr->load() has been deleted by another thread in the meantime. } diff --git a/core/base/src/TROOT.cxx b/core/base/src/TROOT.cxx index c64f3a24a3ff3..24d84bc09432c 100644 --- a/core/base/src/TROOT.cxx +++ b/core/base/src/TROOT.cxx @@ -521,6 +521,9 @@ namespace Internal { /// a hint for ROOT: it will try to satisfy the request if the execution /// scenario allows it. For example, if ROOT is configured to use an external /// scheduler, setting a value for 'numthreads' might not have any effect. + /// + /// \note Use `DisableImplicitMT()` to disable multi-threading (some locks will remain in place as + /// described in EnableThreadSafety()). `EnableImplicitMT(1)` creates a thread-pool of size 1. void EnableImplicitMT(UInt_t numthreads) { #ifdef R__USE_IMT diff --git a/core/base/src/TStyle.cxx b/core/base/src/TStyle.cxx index 546db26c75cae..dd0f711294a26 100644 --- a/core/base/src/TStyle.cxx +++ b/core/base/src/TStyle.cxx @@ -64,9 +64,9 @@ style is created simply by: auto default = new TStyle("Default","Default Style"); ~~~ -The "**Plain**" style can be used if you are working on a monochrome display or +The `Plain` style can be used if you are working on a monochrome display or if you want to get a "conventional" Postscript output. These are the instructions -in the ROOT constructor to create the "Plain*" style. +in the ROOT constructor to create the `Plain` style. ``` auto plain = new TStyle("Plain","Plain Style (no colors/fill areas)"); @@ -150,12 +150,12 @@ TStyle::TStyle() :TNamed() //////////////////////////////////////////////////////////////////////////////// /// Create a new TStyle. /// The following names are reserved to create special styles -/// - `Classic`: the default style set in TStyle::Reset +/// - `Classic`: Similar to 'Default` style set in TStyle::Reset /// - `Plain`: a black&white oriented style /// - `Bold` /// - `Video` /// - `Pub` -/// - `Modern` +/// - `Modern` Used when ROOT starts /// - `ATLAS`: style used by the ATLAS experiment /// - `BELLE2`: style used by the BELLE II experiment /// (see the definition of these styles below). @@ -445,7 +445,7 @@ TStyle::TStyle(const char *name, const char *title) SetLabelOffset(0.015,"z"); SetTickLength(0.03,"x"); - SetTickLength(0.02,"y"); // This way we slighty achive equal length ticks for x and y + SetTickLength(0.02,"y"); // This way we slightly achieve equal length ticks for x and y // use bold lines and markers SetMarkerStyle(20); diff --git a/core/clib/src/mmapsup.c b/core/clib/src/mmapsup.c index 6017afd91b773..daf0109977b96 100644 --- a/core/clib/src/mmapsup.c +++ b/core/clib/src/mmapsup.c @@ -42,13 +42,6 @@ typedef char* caddr_t; #include #endif /* __CYGWIN__ */ -#if defined(R__LINUX) && !defined(R__GLIBC) && !defined(__CYGWIN__) \ - || (defined(__CYGWIN__) && (CYGWIN_VERSION_API_MAJOR > 0 || CYGWIN_VERSION_API_MINOR < 213)) -extern size_t getpagesize PARAMS ((void)); -#else -extern int getpagesize PARAMS ((void)); -#endif - #ifndef SEEK_SET #define SEEK_SET 0 #endif diff --git a/core/clib/src/mvalloc.c b/core/clib/src/mvalloc.c index ef0fdc8196785..e062c2d20d052 100644 --- a/core/clib/src/mvalloc.c +++ b/core/clib/src/mvalloc.c @@ -29,13 +29,6 @@ Boston, MA 02111-1307, USA. */ #include #endif /* __CYGWIN__ */ -#if defined(R__LINUX) && !defined(R__GLIBC) && !defined(__CYGWIN__) \ - || (defined(__CYGWIN__) && (CYGWIN_VERSION_API_MAJOR > 0 || CYGWIN_VERSION_API_MINOR < 213)) -extern size_t getpagesize PARAMS ((void)); -#else -extern int getpagesize PARAMS ((void)); -#endif - #ifdef VMS #undef _SC_PAGE_SIZE #undef _SC_PAGESIZE diff --git a/core/cont/inc/TArrayL.h b/core/cont/inc/TArrayL.h index c51e90b8b5129..9d578d9678b9b 100644 --- a/core/cont/inc/TArrayL.h +++ b/core/cont/inc/TArrayL.h @@ -27,7 +27,7 @@ class TArrayL : public TArray { public: - Long_t *fArray; //[fN] Array of fN longs + Long_t *fArray; //[fN] Array of fN longs TArrayL(); TArrayL(Int_t n); diff --git a/core/cont/inc/TCollectionProxyInfo.h b/core/cont/inc/TCollectionProxyInfo.h index 3787d1311a8df..79a73b1b9a43c 100644 --- a/core/cont/inc/TCollectionProxyInfo.h +++ b/core/cont/inc/TCollectionProxyInfo.h @@ -752,10 +752,6 @@ namespace Detail { } }; - // TODO this can/should go away when we move to new RVec - template <> - struct TCollectionProxyInfo::Type> : TCollectionProxyInfo::Type> {}; - template struct TCollectionProxyInfo::Type > : public TCollectionProxyInfo::Address { typedef Bitset_t Cont_t; diff --git a/core/dictgen/src/rootcling_impl.cxx b/core/dictgen/src/rootcling_impl.cxx index a6e7ab9dfe26e..59547c489721e 100644 --- a/core/dictgen/src/rootcling_impl.cxx +++ b/core/dictgen/src/rootcling_impl.cxx @@ -3801,6 +3801,14 @@ gOptIncludePaths("I", llvm::cl::Prefix, llvm::cl::ZeroOrMore, llvm::cl::desc("Specify an include path."), llvm::cl::cat(gRootclingOptions)); static llvm::cl::list +gOptCompDefaultIncludePaths("compilerI", llvm::cl::Prefix, llvm::cl::ZeroOrMore, + llvm::cl::desc("Specify a compiler default include path, to suppress unneeded `-isystem` arguments."), + llvm::cl::cat(gRootclingOptions)); +static llvm::cl::list +gOptSysIncludePaths("isystem", llvm::cl::ZeroOrMore, + llvm::cl::desc("Specify a system include path."), + llvm::cl::cat(gRootclingOptions)); +static llvm::cl::list gOptPPDefines("D", llvm::cl::Prefix, llvm::cl::ZeroOrMore, llvm::cl::desc("Specify defined macros."), llvm::cl::cat(gRootclingOptions)); @@ -4112,6 +4120,16 @@ int RootClingMain(int argc, for (const std::string &IncludePath : gOptIncludePaths) clingArgs.push_back(std::string("-I") + llvm::sys::path::convert_to_slash(IncludePath)); + for (const std::string &IncludePath : gOptSysIncludePaths) { + // Prevent mentioning compiler default include directories as -isystem + // (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70129) + if (std::find(gOptCompDefaultIncludePaths.begin(), gOptCompDefaultIncludePaths.end(), IncludePath) + == gOptCompDefaultIncludePaths.end()) { + clingArgs.push_back("-isystem"); + clingArgs.push_back(llvm::sys::path::convert_to_slash(IncludePath)); + } + } + for (const std::string &WDiag : gOptWDiags) { const std::string FullWDiag = std::string("-W") + WDiag; // Suppress warning when compiling the dictionary, eg. gcc G__xxx.cxx diff --git a/core/foundation/inc/ROOT/RConfig.hxx b/core/foundation/inc/ROOT/RConfig.hxx index 484eea701ffb5..dea1b93801362 100644 --- a/core/foundation/inc/ROOT/RConfig.hxx +++ b/core/foundation/inc/ROOT/RConfig.hxx @@ -44,10 +44,10 @@ # define R__NULLPTR # endif #else -# if defined(__cplusplus) && (__cplusplus < 201103L) -# error "ROOT requires support for C++11 or higher." +# if defined(__cplusplus) && (__cplusplus < 201402L) +# error "ROOT requires support for C++14 or higher." # if defined(__GNUC__) || defined(__clang__) -# error "Pass `-std=c++11` as compiler argument." +# error "Pass `-std=c++14` as compiler argument." # endif # endif #endif @@ -333,16 +333,14 @@ # endif #endif -#if __cplusplus >= 201402L -# if defined(R__MACOSX) && !defined(MAC_OS_X_VERSION_10_12) - // At least on 10.11, the compiler defines but the c++ library does not provide the size operator delete. - // See for example https://llvm.org/bugs/show_bug.cgi?id=22951 or - // https://github.com/gperftools/gperftools/issues/794. -# elif !defined(__GNUC__) -# define R__SIZEDDELETE -# elif __GNUC__ > 4 -# define R__SIZEDDELETE -# endif +#if defined(R__MACOSX) && !defined(MAC_OS_X_VERSION_10_12) + // At least on 10.11, the compiler defines but the c++ library does not provide the size operator delete. + // See for example https://llvm.org/bugs/show_bug.cgi?id=22951 or + // https://github.com/gperftools/gperftools/issues/794. +#elif !defined(__GNUC__) +# define R__SIZEDDELETE +#elif __GNUC__ > 4 +# define R__SIZEDDELETE #endif /* allows symbols to be hidden from the shared library export symbol table */ diff --git a/core/foundation/inc/ROOT/RNotFn.hxx b/core/foundation/inc/ROOT/RNotFn.hxx index 1597807441da6..2e16da37c9a0f 100644 --- a/core/foundation/inc/ROOT/RNotFn.hxx +++ b/core/foundation/inc/ROOT/RNotFn.hxx @@ -28,7 +28,7 @@ namespace std { -namespace Detail { +namespace __ROOT_noinline { template class not_fn_t { std::decay_t fFun; @@ -55,9 +55,9 @@ public: template -Detail::not_fn_t not_fn(F &&f) +__ROOT_noinline::not_fn_t not_fn(F &&f) { - return Detail::not_fn_t(std::forward(f)); + return __ROOT_noinline::not_fn_t(std::forward(f)); } } diff --git a/core/imt/CMakeLists.txt b/core/imt/CMakeLists.txt index 9fa2ee75d7078..3158c365a7201 100644 --- a/core/imt/CMakeLists.txt +++ b/core/imt/CMakeLists.txt @@ -55,7 +55,7 @@ if(imt) src/TThreadExecutor.cxx ) - target_include_directories(Imt PRIVATE ${TBB_INCLUDE_DIRS}) + target_include_directories(Imt SYSTEM PRIVATE ${TBB_INCLUDE_DIRS}) target_link_libraries(Imt PRIVATE ${TBB_LIBRARIES}) set_target_properties(Imt PROPERTIES COMPILE_FLAGS "${TBB_CXXFLAGS}") diff --git a/core/meta/inc/TInterpreter.h b/core/meta/inc/TInterpreter.h index 909c8346e6010..25ec9d5c13cbf 100644 --- a/core/meta/inc/TInterpreter.h +++ b/core/meta/inc/TInterpreter.h @@ -267,6 +267,8 @@ class TInterpreter : public TNamed { int SetClassAutoloading(int a) const { return SetClassAutoLoading(a); } // Deprecated virtual int SetClassAutoparsing(int) {return 0;}; virtual void SetErrmsgcallback(void * /* p */) const {;} + /// \brief Report diagnostics to the ROOT error handler (see TError.h). + virtual void ReportDiagnosticsToErrorHandler(bool /*enable*/ = true) {} virtual void SetTempLevel(int /* val */) const {;} virtual int UnloadFile(const char * /* path */) const {return 0;} diff --git a/core/metacling/src/CMakeLists.txt b/core/metacling/src/CMakeLists.txt index 8cbbe4ba089ce..104da6be54f28 100644 --- a/core/metacling/src/CMakeLists.txt +++ b/core/metacling/src/CMakeLists.txt @@ -10,9 +10,11 @@ if(MSVC) set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/TClingCallbacks.cxx COMPILE_FLAGS -GR-) + set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/TClingDiagnostics.cxx COMPILE_FLAGS -GR-) set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/TClingRdictModuleFileExtension.cxx COMPILE_FLAGS -GR-) else() set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/TClingCallbacks.cxx COMPILE_FLAGS -fno-rtti) + set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/TClingDiagnostics.cxx COMPILE_FLAGS -fno-rtti) set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/TClingRdictModuleFileExtension.cxx COMPILE_FLAGS -fno-rtti) endif() @@ -25,6 +27,7 @@ ROOT_OBJECT_LIBRARY(MetaCling TClingClassInfo.cxx TClingDataMemberInfo.cxx TClingDeclInfo.cxx + TClingDiagnostics.cxx TClingMemberIter.cxx TClingMethodArgInfo.cxx TClingMethodInfo.cxx @@ -128,33 +131,11 @@ if(MSVC) if("${CMAKE_GENERATOR_PLATFORM}" MATCHES "x64") set(cling_exports ${cling_exports} ??2@YAPEAX_K@Z - ??2@YAPEAX_KPEAX@Z - ??2@YAPEAX_KHPEBDH@Z ??3@YAXPEAX@Z ??3@YAXPEAX_K@Z - ??3@YAXPEAXHPEBDH@Z ??_U@YAPEAX_K@Z - ??_U@YAPEAX_KHPEBDH@Z ??_V@YAXPEAX@Z ??_V@YAXPEAX_K@Z - ??_V@YAXPEAXHPEBDH@Z - ?print@Decl@clang@@QEBAXAEAVraw_ostream@llvm@@I_N@Z - ??6raw_ostream@llvm@@QEAAAEAV01@PEBX@Z - ?decls_begin@DeclContext@clang@@QEBA?AVdecl_iterator@12@XZ - ?errs@llvm@@YAAEAVraw_ostream@1@XZ - ?grow_pod@SmallVectorBase@llvm@@IEAAXPEAX_K1@Z - ?write@raw_ostream@llvm@@QEAAAEAV12@E@Z - ?write@raw_ostream@llvm@@QEAAAEAV12@PEBD_K@Z - ?castFromDeclContext@Decl@clang@@SAPEAV12@PEBVDeclContext@2@@Z - ??1raw_ostream@llvm@@UEAA@XZ - ??1raw_string_ostream@llvm@@UEAA@XZ - ?flush_nonempty@raw_ostream@llvm@@AEAAXXZ - ?getASTContext@Decl@clang@@QEBAAEAVASTContext@2@XZ - ?preferred_buffer_size@raw_ostream@llvm@@MEBA_KXZ - ?write_impl@raw_string_ostream@llvm@@EEAAXPEBD_K@Z - ?castToDeclContext@Decl@clang@@SAPEAVDeclContext@2@PEBV12@@Z - ?classof@DeclContext@clang@@SA_NPEBVDecl@2@@Z - ?_Facet_Register@std@@YAXPEAV_Facet_base@1@@Z ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@H@Z ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@M@Z ??6?$basic_ostream@DU?$char_traits@D@std@@@std@@QEAAAEAV01@N@Z @@ -163,6 +144,7 @@ if(MSVC) ??$?6U?$char_traits@D@std@@@std@@YAAEAV?$basic_ostream@DU?$char_traits@D@std@@@0@AEAV10@D@Z ??$?6U?$char_traits@D@std@@@std@@YAAEAV?$basic_ostream@DU?$char_traits@D@std@@@0@AEAV10@PEBD@Z ??$endl@DU?$char_traits@D@std@@@std@@YAAEAV?$basic_ostream@DU?$char_traits@D@std@@@0@AEAV10@@Z + ?_Facet_Register@std@@YAXPEAV_Facet_base@1@@Z ) if($) set(cling_exports ${cling_exports} ??$dyn_cast@VValueDecl@clang@@$$CBVDecl@2@@llvm@@YAPEBVValueDecl@clang@@PEBVDecl@2@@Z) diff --git a/core/metacling/src/TCling.cxx b/core/metacling/src/TCling.cxx index 300955e471113..05fb268641def 100644 --- a/core/metacling/src/TCling.cxx +++ b/core/metacling/src/TCling.cxx @@ -40,6 +40,7 @@ clang/LLVM technology. #include "TClassEdit.h" #include "TClassTable.h" #include "TClingCallbacks.h" +#include "TClingDiagnostics.h" #include "TBaseClass.h" #include "TDataMember.h" #include "TMemberInspector.h" @@ -3393,6 +3394,15 @@ void TCling::RegisterLoadedSharedLibrary(const char* filename) || strstr(filename, "/usr/lib/libAudioToolboxUtility") || strstr(filename, "/usr/lib/liboah") || strstr(filename, "/usr/lib/libRosetta") + || strstr(filename, "/usr/lib/libCoreEntitlements") + // These are candidates for suppression, too: + // -lfakelink -lapple_nghttp2 -lnetwork -lsqlite3 -lenergytrace -lCoreEntitlements + // -lMobileGestalt -lcoretls -lcoretls_cfhelpers -lxar.1 -lcompression -larchive.2 + // -lxml2.2 -lpcap.A -ldns_services -llzma.5 -lbz2.1.0 -liconv.2 -lcharset.1 + // -lCheckFix -lmecabra -lmecab -lgermantok -lThaiTokenizer -lChineseTokenizer + // -lcmph -lutil -lapp_launch_measurement -lxslt.1 -lspindump -late -lexpat.1 + // -lAudioStatistics -lSMC -lperfcheck -lmis -lIOReport -lheimdal-asn1 + // "cannot link directly with dylib/framework, your binary is not an allowed client of // /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/ // SDKs/MacOSX.sdk/usr/lib/libAudioToolboxUtility.tbd for architecture x86_64 @@ -7577,6 +7587,28 @@ void TCling::SetErrmsgcallback(void* p) const #endif } +void TCling::ReportDiagnosticsToErrorHandler(bool enable) +{ + if (enable) { + auto consumer = new TClingDelegateDiagnosticPrinter( + &fInterpreter->getDiagnostics().getDiagnosticOptions(), + fInterpreter->getCI()->getLangOpts(), + [] (clang::DiagnosticsEngine::Level Level, const std::string &Info) { + if (Level == clang::DiagnosticsEngine::Warning) { + ::Warning("cling", "%s", Info.c_str()); + } else if (Level == clang::DiagnosticsEngine::Error + || Level == clang::DiagnosticsEngine::Fatal) { + ::Error("cling", "%s", Info.c_str()); + } else { + ::Info("cling", "%s", Info.c_str()); + } + }); + fInterpreter->replaceDiagnosticConsumer(consumer, /*Own=*/true); + } else { + fInterpreter->replaceDiagnosticConsumer(nullptr); + } +} + //////////////////////////////////////////////////////////////////////////////// /// Create / close a scope for temporaries. No-op for cling; use diff --git a/core/metacling/src/TCling.h b/core/metacling/src/TCling.h index b7db7d3c960e2..ec71533c9c42e 100644 --- a/core/metacling/src/TCling.h +++ b/core/metacling/src/TCling.h @@ -354,6 +354,7 @@ class TCling final : public TInterpreter { virtual int SetClassAutoparsing(int) ; Bool_t IsAutoParsingSuspended() const { return fIsAutoParsingSuspended; } virtual void SetErrmsgcallback(void* p) const; + virtual void ReportDiagnosticsToErrorHandler(bool enable = true); virtual void SetTempLevel(int val) const; virtual int UnloadFile(const char* path) const; diff --git a/core/metacling/src/TClingCallFunc.cxx b/core/metacling/src/TClingCallFunc.cxx index 8bc0227b150a1..5e2135246964f 100644 --- a/core/metacling/src/TClingCallFunc.cxx +++ b/core/metacling/src/TClingCallFunc.cxx @@ -982,7 +982,7 @@ void TClingCallFunc::make_narg_call_with_return(const unsigned N, const string & // new (ret) (return_type) ((class_name*)obj)->func(args...); // } // else { - // ((class_name*)obj)->func(args...); + // (void)(((class_name*)obj)->func(args...)); // } // const FunctionDecl *FD = GetDecl(); @@ -1085,8 +1085,9 @@ void TClingCallFunc::make_narg_call_with_return(const unsigned N, const string & for (int i = 0; i < indent_level; ++i) { callbuf << kIndentString; } + callbuf << "(void)("; make_narg_call(type_name, N, typedefbuf, callbuf, class_name, indent_level); - callbuf << ";\n"; + callbuf << ");\n"; for (int i = 0; i < indent_level; ++i) { callbuf << kIndentString; } diff --git a/core/metacling/src/TClingCallbacks.cxx b/core/metacling/src/TClingCallbacks.cxx index 2bcb0cbb7a3e9..0605cc6473e9f 100644 --- a/core/metacling/src/TClingCallbacks.cxx +++ b/core/metacling/src/TClingCallbacks.cxx @@ -728,7 +728,7 @@ bool TClingCallbacks::tryResolveAtRuntimeInternal(LookupResult &R, Scope *S) { bool TClingCallbacks::shouldResolveAtRuntime(LookupResult& R, Scope* S) { if (m_IsRuntime) - return false; + return false; if (R.getLookupKind() != Sema::LookupOrdinaryName) return false; @@ -746,6 +746,16 @@ bool TClingCallbacks::shouldResolveAtRuntime(LookupResult& R, Scope* S) { if (!COpts.DynamicScoping) return false; + auto &PP = R.getSema().PP; + // In `foo bar`, `foo` is certainly a type name and must not be resolved. We + // cannot rely on `PP.LookAhead(0)` as the parser might have already consumed + // some tokens. + SourceLocation LocAfterIdent = PP.getLocForEndOfToken(R.getNameLoc()); + Token LookAhead0; + PP.getRawToken(LocAfterIdent, LookAhead0, /*IgnoreWhiteSpace=*/true); + if (LookAhead0.is(tok::raw_identifier)) + return false; + // FIXME: Figure out better way to handle: // C++ [basic.lookup.classref]p1: // In a class member access expression (5.2.5), if the . or -> token is diff --git a/core/metacling/src/TClingDiagnostics.cxx b/core/metacling/src/TClingDiagnostics.cxx new file mode 100644 index 0000000000000..511b4c1f3735d --- /dev/null +++ b/core/metacling/src/TClingDiagnostics.cxx @@ -0,0 +1,29 @@ +// @(#)root/core/metacling:$Id$ +// Author: Javier Lopez-Gomez 16/07/2021 + +/************************************************************************* + * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#include "TClingDiagnostics.h" + +TClingDelegateDiagnosticPrinter::TClingDelegateDiagnosticPrinter +(clang::DiagnosticOptions *DiagOpts, clang::LangOptions &LangOpts, handler_t fn) + : TextDiagnosticPrinter(fOS, DiagOpts), fOS(fOS_out), fHandler(fn) +{ + // Required to initialize the internal `clang::TextDiagnostic` instance. + TextDiagnosticPrinter::BeginSourceFile(LangOpts, nullptr); +} + +void +TClingDelegateDiagnosticPrinter::HandleDiagnostic(clang::DiagnosticsEngine::Level Level, + const clang::Diagnostic &Info) +{ + fOS_out.clear(); + TextDiagnosticPrinter::HandleDiagnostic(Level, Info); + fHandler(Level, fOS.str()); +} diff --git a/core/metacling/src/TClingDiagnostics.h b/core/metacling/src/TClingDiagnostics.h new file mode 100644 index 0000000000000..62680f0f1ce85 --- /dev/null +++ b/core/metacling/src/TClingDiagnostics.h @@ -0,0 +1,45 @@ +// @(#)root/core/metacling:$Id$ +// Author: Javier Lopez-Gomez 16/07/2021 + +/************************************************************************* + * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#ifndef ROOT_TClingDiagnostics +#define ROOT_TClingDiagnostics + +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "llvm/Support/raw_ostream.h" + +#include + +namespace clang { + class LangOptions; +} + +/// \brief Uses `clang::TextDiagnosticPrinter` to format diagnostics, which +/// are then passed to a user-specified function. +/// +class TClingDelegateDiagnosticPrinter : public clang::TextDiagnosticPrinter { +public: + typedef void (*handler_t)(clang::DiagnosticsEngine::Level Level, + const std::string &Info); +private: + std::string fOS_out; + llvm::raw_string_ostream fOS; + handler_t fHandler; + +public: + TClingDelegateDiagnosticPrinter(clang::DiagnosticOptions *DiagOpts, + clang::LangOptions &LangOpts, handler_t fn); + ~TClingDelegateDiagnosticPrinter() override = default; + + void HandleDiagnostic(clang::DiagnosticsEngine::Level Level, + const clang::Diagnostic &Info) override; +}; + +#endif // ROOT_TClingDiagnostics diff --git a/core/metacling/test/TClingCallFuncTests.cxx b/core/metacling/test/TClingCallFuncTests.cxx index d813cb1d0b413..7e1f5c3fbd26a 100644 --- a/core/metacling/test/TClingCallFuncTests.cxx +++ b/core/metacling/test/TClingCallFuncTests.cxx @@ -1,3 +1,4 @@ +#include "ROOTUnitTestSupport.h" #include "TInterpreter.h" #include "gtest/gtest.h" @@ -259,3 +260,36 @@ TEST(TClingCallFunc, DISABLED_OverloadedTemplate) } )cpp"); } + +TEST(TClingCallFunc, FunctionWrapperNodiscard) +{ + gInterpreter->Declare(R"cpp( + struct TClingCallFunc_Nodiscard1 { + #if __cplusplus >= 201703L + [[nodiscard]] + #endif + bool foo(int) { return false; } + }; + )cpp"); + + ClassInfo_t *FooNamespace = gInterpreter->ClassInfo_Factory("TClingCallFunc_Nodiscard1"); + CallFunc_t *mc = gInterpreter->CallFunc_Factory(); + Longptr_t offset = 0; + + gInterpreter->CallFunc_SetFuncProto(mc, FooNamespace, "foo", "int", &offset); + std::string wrapper = gInterpreter->CallFunc_GetWrapperCode(mc); + + { + using ::testing::Not; + using ::testing::HasSubstr; + FilterDiagsRAII RAII([] (int /*level*/, Bool_t /*abort*/, + const char * /*location*/, const char *msg) { + EXPECT_THAT(msg, Not(HasSubstr("-Wunused-result"))); + }); + ASSERT_TRUE(gInterpreter->Declare(wrapper.c_str())); + } + + // Cleanup + gInterpreter->CallFunc_Delete(mc); + gInterpreter->ClassInfo_Delete(FooNamespace); +} diff --git a/core/rint/inc/TRint.h b/core/rint/inc/TRint.h index 5706cd94d8aa9..3a730945c1cf2 100644 --- a/core/rint/inc/TRint.h +++ b/core/rint/inc/TRint.h @@ -38,6 +38,7 @@ class TRint : public TApplication { Bool_t fInterrupt; // if true macro execution will be stopped Int_t fCaughtSignal; // TRint just caught a signal TFileHandler *fInputHandler; // terminal input handler + Bool_t fBackslashContinue{};// whether the last line ended with '\' TRint(const TRint&) = delete; TRint& operator=(const TRint&) = delete; diff --git a/core/rint/src/TRint.cxx b/core/rint/src/TRint.cxx index b5b0e6a6e830a..a07bb49d1f61e 100644 --- a/core/rint/src/TRint.cxx +++ b/core/rint/src/TRint.cxx @@ -41,7 +41,7 @@ #include "TTabCom.h" #include #include - +#include #include "Getline.h" #include "strlcpy.h" #include "snprintf.h" @@ -146,6 +146,16 @@ TRint::TRint(const char *appClassName, Int_t *argc, char **argv, void *options, TApplication(appClassName, argc, argv, options, numOptions), fCaughtSignal(-1) { + + if (*argc > 1) { + // Early exit if there are remaining unrecognized options + for (auto n = 1; n < *argc; n++) { + std::cerr << "root: unrecognized option '" << argv[n] << "'\n"; + } + std::cerr << "Try 'root --help' for more information.\n"; + TApplication::Terminate(0); + } + fNcmd = 0; fDefaultPrompt = "root [%d] "; fInterrupt = kFALSE; @@ -747,8 +757,11 @@ Longptr_t TRint::ProcessLineNr(const char* filestem, const char *line, Int_t *e if (!error) error = &err; if (line && line[0] != '.') { - TString lineWithNr = TString::Format("#line 1 \"%s%d\"\n", filestem, fNcmd - 1); - int res = ProcessLine(lineWithNr + line, kFALSE, error); + TString input; + if (!fBackslashContinue) + input += TString::Format("#line 1 \"%s%d\"\n", filestem, fNcmd - 1); + input += line; + int res = ProcessLine(input, kFALSE, error); if (*error == TInterpreter::kProcessing) { if (!fNonContinuePrompt.Length()) fNonContinuePrompt = fDefaultPrompt; @@ -757,6 +770,10 @@ Longptr_t TRint::ProcessLineNr(const char* filestem, const char *line, Int_t *e SetPrompt(fNonContinuePrompt); fNonContinuePrompt.Clear(); } + std::string_view sv(line); + auto lastNonSpace = sv.find_last_not_of(" \t"); + fBackslashContinue = (lastNonSpace != std::string_view::npos + && sv[lastNonSpace] == '\\'); return res; } if (line && line[0] == '.' && line[1] == '@') { diff --git a/core/rint/test/CMakeLists.txt b/core/rint/test/CMakeLists.txt index c72b38e164e23..4518dae395996 100644 --- a/core/rint/test/CMakeLists.txt +++ b/core/rint/test/CMakeLists.txt @@ -5,3 +5,5 @@ # For the list of contributors see $ROOTSYS/README/CREDITS. ROOT_ADD_GTEST(TTabComTests TTabComTests.cxx LIBRARIES Rint) +ROOT_ADD_GTEST(TRintTests TRintTests.cxx LIBRARIES Rint) + diff --git a/core/rint/test/TRintTests.cxx b/core/rint/test/TRintTests.cxx new file mode 100644 index 0000000000000..8b292808ecd82 --- /dev/null +++ b/core/rint/test/TRintTests.cxx @@ -0,0 +1,36 @@ +#include +#include + +#include "gtest/gtest.h" + +#include "TRint.h" + +using testing::internal::CaptureStderr; +using testing::internal::GetCapturedStderr; + +TEST(TRint, UnrecognizedOptions) +{ + // Create array of options. + // We need to create it as a dynamic array for the following reasons: + // - TRint constructor accepts a char** so we construct directly that type + // - TRint will modify this array, removing recognized options and leaving + // only unrecognized ones, so we can't create an std::vector and pass its + // data to TRint directly. + int argc{4}; + char e1[]{"-q"}; + char e2[]{"-z"}; + char e3[]{"--nonexistingoption"}; + char e4[]{"-b"}; + char *argv[]{e1, e2, e3, e4}; + + CaptureStderr(); + // Unrecognized options will be printed to stderr + TRint app{"App", &argc, argv}; + std::string trinterr = GetCapturedStderr(); + + const std::string expected{"root: unrecognized option '-z'\n" + "root: unrecognized option '--nonexistingoption'\n" + "Try 'root --help' for more information.\n"}; + + EXPECT_EQ(trinterr, expected); +} diff --git a/core/thread/inc/ThreadLocalStorage.h b/core/thread/inc/ThreadLocalStorage.h index 51ee894adf17d..5e690567e7d9d 100644 --- a/core/thread/inc/ThreadLocalStorage.h +++ b/core/thread/inc/ThreadLocalStorage.h @@ -13,7 +13,7 @@ * thread_local or * Pthread-Keys * depending on the (configure-set) CPP-variables R__HAS___THREAD, - * R__HAS_DECLSPEC_THREAD, R__HAS_THREAD_LOCAL or R__HAS_PTHREAD. + * R__HAS_DECLSPEC_THREAD or R__HAS_PTHREAD. * * Use the macros TTHREAD_TLS_DECLARE, TTHREAD_TLS_INIT, and the * getters and setters TTHREAD_TLS_GET and TTHREAD_TLS_GET @@ -36,8 +36,7 @@ * func (a, TTHREAD_TLS_SET(int, my_var, 5)); * as we do not use the gcc-extension of returning macro-values. * - * C++11 requires the implementation of the thread_local storage but - * a few platforms do not yet implement it. + * C++11 requires the implementation of the thread_local storage. * * For simple type use: * TTHREAD_TLS(int) varname; @@ -82,34 +81,6 @@ # define R__HAS_DECLSPEC_THREAD #endif -#if __cplusplus >= 201103L - -// Clang 3.4 also support SD-6 (feature test macros __cpp_*), but no thread local macro -# if defined(__clang__) - -# if __has_feature(cxx_thread_local) - // thread_local was added in Clang 3.3 - // Still requires libstdc++ from GCC 4.8 - // For that __GLIBCXX__ isn't good enough - // Also the MacOS build of clang does *not* support thread local yet. -# define R__HAS_THREAD_LOCAL -# else -# define R__HAS___THREAD -# endif - -# elif defined(__INTEL_COMPILER) -# define R__HAS__THREAD - -# elif defined(__GNUG__) && (__GNUC__ <= 4 && __GNUC_MINOR__ < 8) - // The C++11 thread_local keyword is supported in GCC only since 4.8 -# define R__HAS___THREAD -# else -# define R__HAS_THREAD_LOCAL -# endif - -#endif - - #ifdef __cplusplus // Note that the order is relevant, more than one of the flag might be @@ -121,7 +92,7 @@ # define TTHREAD_TLS_ARRAY(type,size,name) static type name[size] # define TTHREAD_TLS_PTR(name) &name -#elif defined(R__HAS_THREAD_LOCAL) +#else # define TTHREAD_TLS(type) thread_local type # define TTHREAD_TLS_ARRAY(type,size,name) thread_local type name[size] @@ -131,124 +102,6 @@ # define TTHREAD_TLS_DECL_ARG(type, name, arg) thread_local type name(arg) # define TTHREAD_TLS_DECL_ARG2(type, name, arg1, arg2) thread_local type name(arg1,arg2) -#elif defined(R__HAS___THREAD) - -# define TTHREAD_TLS(type) static __thread type -# define TTHREAD_TLS_ARRAY(type,size,name) static __thread type name[size] -# define TTHREAD_TLS_PTR(name) &name - -#elif defined(R__HAS_DECLSPEC_THREAD) - -# define TTHREAD_TLS(type) static __declspec(thread) type -# define TTHREAD_TLS_ARRAY(type,size,name) static __declspec(thread) type name[size] -# define TTHREAD_TLS_PTR(name) &name - -#elif defined(R__HAS_PTHREAD) - -#include -#include - -template class TThreadTLSWrapper -{ -private: - pthread_key_t fKey; - type fInitValue; - - static void key_delete(void *arg) { - assert (NULL != arg); - delete (type*)(arg); - } - -public: - - TThreadTLSWrapper() : fInitValue() { - - pthread_key_create(&(fKey), key_delete); - } - - TThreadTLSWrapper(const type &value) : fInitValue(value) { - - pthread_key_create(&(fKey), key_delete); - } - - ~TThreadTLSWrapper() { - pthread_key_delete(fKey); - } - - type& get() { - void *ptr = pthread_getspecific(fKey); - if (!ptr) { - ptr = new type(fInitValue); - assert (NULL != ptr); - (void) pthread_setspecific(fKey, ptr); - } - return *(type*)ptr; - } - - type& operator=(const type &in) { - type &ptr = get(); - ptr = in; - return ptr; - } - - operator type&() { - return get(); - } - -}; - -template class TThreadTLSArrayWrapper -{ -private: - pthread_key_t fKey; - - static void key_delete(void *arg) { - assert (NULL != arg); - delete [] (type*)(arg); - } - -public: - - TThreadTLSArrayWrapper() { - - pthread_key_create(&(fKey), key_delete); - } - - ~TThreadTLSArrayWrapper() { - pthread_key_delete(fKey); - } - - type* get() { - void *ptr = pthread_getspecific(fKey); - if (!ptr) { - ptr = new type[size]; - assert (NULL != ptr); - (void) pthread_setspecific(fKey, ptr); - } - return (type*)ptr; - } - -// type& operator=(const type &in) { -// type &ptr = get(); -// ptr = in; -// return ptr; -// } -// - operator type*() { - return get(); - } - -}; - -# define TTHREAD_TLS(type) static TThreadTLSWrapper -# define TTHREAD_TLS_ARRAY(type,size,name) static TThreadTLSArrayWrapper name; -# define TTHREAD_TLS_PTR(name) &(name.get()) -# define TTHREAD_TLS_OBJ(index,type,name) type &name( TTHREAD_TLS_INIT() ) - -#else - -#error "No Thread Local Storage (TLS) technology for this platform specified." - #endif // Available on all platforms @@ -315,15 +168,11 @@ T &TTHREAD_TLS_INIT(ArgType arg) { #else // __cplusplus -#if defined(R__HAS_THREAD_LOCAL) - -# define TTHREAD_TLS_DECLARE(type,name) -# define TTHREAD_TLS_INIT(type,name,value) thread_local type name = (value) -# define TTHREAD_TLS_SET(type,name,value) name = (value) -# define TTHREAD_TLS_GET(type,name) (name) -# define TTHREAD_TLS_FREE(name) +// TODO: Evaluate using thread_local / _Thread_local from C11 instead of +// platform-specific solutions such as __thread, __declspec(thread) or the +// pthreads API functions. -#elif defined(R__HAS___THREAD) +#if defined(R__HAS___THREAD) # define TTHREAD_TLS_DECLARE(type,name) # define TTHREAD_TLS_INIT(type,name,value) static __thread type name = (value) diff --git a/core/unix/src/TUnixSystem.cxx b/core/unix/src/TUnixSystem.cxx index 25dfe1cf87c6a..dcfa6873110a6 100644 --- a/core/unix/src/TUnixSystem.cxx +++ b/core/unix/src/TUnixSystem.cxx @@ -214,7 +214,6 @@ extern "C" { // FPE handling includes #if (defined(R__LINUX) && !defined(R__WINGCC)) -#include #include #include // for prctl() function used in StackTrace() #endif diff --git a/documentation/doxygen/Doxyfile b/documentation/doxygen/Doxyfile index 0d7238ff8958f..495b7533399e9 100644 --- a/documentation/doxygen/Doxyfile +++ b/documentation/doxygen/Doxyfile @@ -1,4 +1,4 @@ -# Doxyfile 1.9.0 +# Doxyfile 1.9.3 # This file describes the settings to be used by the documentation system # doxygen (www.doxygen.org) for a project. @@ -93,14 +93,6 @@ ALLOW_UNICODE_NAMES = NO OUTPUT_LANGUAGE = English -# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all generated output in the proper direction. -# Possible values are: None, LTR, RTL and Context. -# The default value is: None. - -OUTPUT_TEXT_DIRECTION = None - # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member # descriptions after the members that are listed in the file and class # documentation (similar to Javadoc). Set to NO to disable this. @@ -258,18 +250,18 @@ TAB_SIZE = 3 # the documentation. An alias has the form: # name=value # For example adding -# "sideeffect=@par Side Effects:\n" +# "sideeffect=@par Side Effects:^^" # will allow you to put the command \sideeffect (or @sideeffect) in the # documentation, which will result in a user-defined paragraph with heading -# "Side Effects:". You can put \n's in the value part of an alias to insert -# newlines (in the resulting output). You can put ^^ in the value part of an -# alias to insert a newline as if a physical newline was in the original file. -# When you need a literal { or } or , in the value part of an alias you have to -# escape them by means of a backslash (\), this can lead to conflicts with the -# commands \{ and \} for these it is advised to use the version @{ and @} or use -# a double escape (\\{ and \\}) +# "Side Effects:". Note that you cannot put \n's in the value part of an alias +# to insert newlines (in the resulting output). You can put ^^ in the value part +# of an alias to insert a newline as if a physical newline was in the original +# file. When you need a literal { or } or , in the value part of an alias you +# have to escape them by means of a backslash (\), this can lead to conflicts +# with the commands \{ and \} for these it is advised to use the version @{ and +# @} or use a double escape (\\{ and \\}) -ALIASES = +ALIASES = "legacy{1}=\htmlonly

Legacy Code

\1 is a legacy interface: it is not recommended to use it in new code. There will be no bug fixes nor new developments.
\endhtmlonly" # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources # only. Doxygen will then generate output that is more tailored for C. For @@ -312,8 +304,8 @@ OPTIMIZE_OUTPUT_SLICE = NO # extension. Doxygen has a built-in mapping, but you can override or extend it # using this tag. The format is ext=language, where ext is a file extension, and # language is one of the parsers supported by doxygen: IDL, Java, JavaScript, -# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice, VHDL, -# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: +# Csharp (C#), C, C++, Lex, D, PHP, md (Markdown), Objective-C, Python, Slice, +# VHDL, Fortran (fixed format Fortran: FortranFixed, free formatted Fortran: # FortranFree, unknown formatted Fortran: Fortran. In the later case the parser # tries to guess whether the code is fixed or free formatted code, this is the # default for Fortran type files). For instance to make doxygen treat .inc files @@ -323,7 +315,10 @@ OPTIMIZE_OUTPUT_SLICE = NO # Note: For files without extension you can use no_extension as a placeholder. # # Note that for custom extensions you also need to set FILE_PATTERNS otherwise -# the files are not read by doxygen. +# the files are not read by doxygen. When specifying no_extension you should add +# * to the FILE_PATTERNS. +# +# Note see also the list of default file extension mappings. EXTENSION_MAPPING = h=C++ \ icc=C++ \ @@ -465,7 +460,7 @@ LOOKUP_CACHE_SIZE = 4 # than 0 to get more control over the balance between CPU load and processing # speed. At this moment only the input processing can be done using multiple # threads. Since this is still an experimental feature the default is set to 1, -# which efficively disables parallel processing. Please report any issues you +# which effectively disables parallel processing. Please report any issues you # encounter. Generating dot graphs in parallel is controlled by the # DOT_NUM_THREADS setting. # Minimum value: 0, maximum value: 32, default value: 1. @@ -535,6 +530,13 @@ EXTRACT_LOCAL_METHODS = NO EXTRACT_ANON_NSPACES = NO +# If this flag is set to YES, the name of an unnamed parameter in a declaration +# will be determined by the corresponding definition. By default unnamed +# parameters remain unnamed in the output. +# The default value is: YES. + +RESOLVE_UNNAMED_PARAMS = YES + # If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all # undocumented members inside documented classes or files. If set to NO these # members will be included in the various overviews, but no documentation @@ -572,11 +574,18 @@ HIDE_IN_BODY_DOCS = YES INTERNAL_DOCS = NO -# If the CASE_SENSE_NAMES tag is set to NO then doxygen will only generate file -# names in lower-case letters. If set to YES, upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# (including Cygwin) and Mac users are advised to set this option to NO. +# With the correct setting of option CASE_SENSE_NAMES doxygen will better be +# able to match the capabilities of the underlying filesystem. In case the +# filesystem is case sensitive (i.e. it supports files in the same directory +# whose names only differ in casing), the option must be set to YES to properly +# deal with such files in case they appear in the input. For filesystems that +# are not case sensitive the option should be be set to NO to properly deal with +# output files written for symbols that only differ in casing, such as for two +# classes, one named CLASS and the other named Class, and to also support +# references to files without having to specify the exact matching casing. On +# Windows (including Cygwin) and MacOS, users should typically set this option +# to NO, whereas on Linux or other Unix flavors it should typically be set to +# YES. # The default value is: system dependent. CASE_SENSE_NAMES = YES @@ -595,6 +604,12 @@ HIDE_SCOPE_NAMES = NO HIDE_COMPOUND_REFERENCE= NO +# If the SHOW_HEADERFILE tag is set to YES then the documentation for a class +# will show which file needs to be included to use the class. +# The default value is: YES. + +SHOW_HEADERFILE = YES + # If the SHOW_INCLUDE_FILES tag is set to YES then doxygen will put a list of # the files that are included by a file in the documentation of that file. # The default value is: YES. @@ -752,7 +767,8 @@ FILE_VERSION_FILTER = # output files in an output format independent way. To create the layout file # that represents doxygen's defaults, run doxygen with the -l option. You can # optionally specify a file name after the option, if omitted DoxygenLayout.xml -# will be used as the name of the layout file. +# will be used as the name of the layout file. See also section "Changing the +# layout of pages" for information. # # Note that if you run doxygen from a directory containing a file called # DoxygenLayout.xml, doxygen will parse it automatically even if the LAYOUT_FILE @@ -798,24 +814,35 @@ WARNINGS = YES WARN_IF_UNDOCUMENTED = YES # If the WARN_IF_DOC_ERROR tag is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some parameters -# in a documented function, or documenting parameters that don't exist or using -# markup commands wrongly. +# potential errors in the documentation, such as documenting some parameters in +# a documented function twice, or documenting parameters that don't exist or +# using markup commands wrongly. # The default value is: YES. WARN_IF_DOC_ERROR = YES +# If WARN_IF_INCOMPLETE_DOC is set to YES, doxygen will warn about incomplete +# function parameter documentation. If set to NO, doxygen will accept that some +# parameters have no documentation without warning. +# The default value is: YES. + +WARN_IF_INCOMPLETE_DOC = YES + # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that # are documented, but have no documentation for their parameters or return -# value. If set to NO, doxygen will only warn about wrong or incomplete -# parameter documentation, but not about the absence of documentation. If -# EXTRACT_ALL is set to YES then this flag will automatically be disabled. +# value. If set to NO, doxygen will only warn about wrong parameter +# documentation, but not about the absence of documentation. If EXTRACT_ALL is +# set to YES then this flag will automatically be disabled. See also +# WARN_IF_INCOMPLETE_DOC # The default value is: NO. WARN_NO_PARAMDOC = NO # If the WARN_AS_ERROR tag is set to YES then doxygen will immediately stop when -# a warning is encountered. +# a warning is encountered. If the WARN_AS_ERROR tag is set to FAIL_ON_WARNINGS +# then doxygen will continue running as if WARN_AS_ERROR tag is set to NO, but +# at the end of the doxygen process doxygen will return with a non-zero status. +# Possible values are: NO, YES and FAIL_ON_WARNINGS. # The default value is: NO. WARN_AS_ERROR = NO @@ -845,89 +872,15 @@ WARN_LOGFILE = # directories like /usr/src/myproject. Separate the files or directories with # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. +# The INPUT tag is defined in the file Doxyfile_INPUT created by the script makeinput.sh -INPUT = ./mainpage.md \ - ../../core/base/ \ - ../../core/dictgen/ \ - ../../core/cont/ \ - ../../core/foundation/ \ - ../../core/gui/ \ - ../../core/macosx/ \ - ../../core/meta/ \ - ../../core/metacling/ \ - ../../core/clingutils/ \ - ../../core/multiproc/ \ - ../../core/rint/ \ - ../../core/thread/ \ - ../../core/unix/ \ - ../../core/winnt/ \ - ../../core/imt/ \ - ../../core/zip/inc/Compression.h \ - ../../geom/ \ - ../../graf2d/asimage/ \ - ../../graf2d/cocoa/ \ - ../../graf2d/fitsio/ \ - ../../graf2d/gpad/ \ - ../../graf2d/gpadv7/ \ - ../../graf2d/graf/ \ - ../../graf2d/gviz/ \ - ../../graf2d/postscript/ \ - ../../graf2d/quartz/ \ - ../../graf2d/win32gdk/ \ - ../../graf2d/x11/ \ - ../../graf2d/x11ttf/ \ - ../../graf3d/eve/ \ - ../../graf3d/eve7/ \ - ../../graf3d/g3d/ \ - ../../graf3d/gl/ \ - ../../graf3d/gviz3d/ \ - ../../gui/ \ - ../../hist/ \ - ../../html/ \ - ../../io/doc/TFile \ - ../../io/dcache/ \ - ../../io/gfal/ \ - ../../io/io/ \ - ../../io/sql/ \ - ../../io/xml/ \ - ../../io/xmlparser/ \ - ../../main/src/hadd.cxx \ - ../../math/ \ - ../../montecarlo/ \ - ../../net/alien/ \ - ../../net/auth/ \ - ../../net/davix/ \ - ../../net/http/ \ - ../../net/monalisa/ \ - ../../net/net/ \ - ../../net/netx/ \ - ../../net/netxng/ \ - ../../proof/ \ - ../../tmva/ \ - ../../roofit/ \ - ../../tree/ \ - ../../sql/ \ - ../../tutorials/ \ - ../../bindings/tpython/ \ - ../../bindings/pyroot/ \ - ../../bindings/r/ -# ../../core/clib/ -# ../../core/lzma/ -# ../../core/newdelete/ -# ../../core/textinput/ -# ../../graf2d/mathtext/ -# ../../graf3d/ftgl/ -# ../../graf3d/glew/ -# ../../graf3d/x3d/ -# ../../net/rootd/ -# ../../net/rpdutils/ - +@INCLUDE = Doxyfile_INPUT # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses # libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. +# documentation (see: +# https://www.gnu.org/software/libiconv/) for the list of possible encodings. # The default value is: UTF-8. INPUT_ENCODING = UTF-8 @@ -940,12 +893,14 @@ INPUT_ENCODING = UTF-8 # need to set EXTENSION_MAPPING for the extension otherwise the files are not # read by doxygen. # +# Note the list of default checked file patterns might differ from the list of +# default file extension mappings. +# # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment), -# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen -# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, +# *.hh, *.hxx, *.hpp, *.h++, *.l, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, +# *.inc, *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C +# comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f18, *.f, *.for, *.vhd, # *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = *.c \ @@ -1223,13 +1178,6 @@ VERBATIM_HEADERS = YES ALPHABETICAL_INDEX = YES -# The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in -# which the alphabetical index list will be split. -# Minimum value: 1, maximum value: 20, default value: 5. -# This tag requires that the tag ALPHABETICAL_INDEX is set to YES. - -COLS_IN_ALPHA_INDEX = 1 - # In case all classes in a project start with a common prefix, all classes will # be put under the same header in the alphabetical index. The IGNORE_PREFIX tag # can be used to specify a prefix (or a list of prefixes) that should be ignored @@ -1330,7 +1278,7 @@ HTML_EXTRA_FILES = ./rootlogo_s.gif \ # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen # will adjust the colors in the style sheet and background images according to -# this color. Hue is specified as an angle on a colorwheel, see +# this color. Hue is specified as an angle on a color-wheel, see # https://en.wikipedia.org/wiki/Hue for more information. For instance the value # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300 # purple, and 360 is red again. @@ -1340,7 +1288,7 @@ HTML_EXTRA_FILES = ./rootlogo_s.gif \ HTML_COLORSTYLE_HUE = 220 # The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of the colors -# in the HTML output. For a value of 0 the output will use grayscales only. A +# in the HTML output. For a value of 0 the output will use gray-scales only. A # value of 255 will produce the most vivid colors. # Minimum value: 0, maximum value: 255, default value: 100. # This tag requires that the tag GENERATE_HTML is set to YES. @@ -1401,10 +1349,11 @@ HTML_INDEX_NUM_ENTRIES = 100 # If the GENERATE_DOCSET tag is set to YES, additional index files will be # generated that can be used as input for Apple's Xcode 3 integrated development -# environment (see: https://developer.apple.com/xcode/), introduced with OSX -# 10.5 (Leopard). To create a documentation set, doxygen will generate a -# Makefile in the HTML output directory. Running make will produce the docset in -# that directory and running make install will install the docset in +# environment (see: +# https://developer.apple.com/xcode/), introduced with OSX 10.5 (Leopard). To +# create a documentation set, doxygen will generate a Makefile in the HTML +# output directory. Running make will produce the docset in that directory and +# running make install will install the docset in # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at # startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy # genXcode/_index.html for more information. @@ -1421,6 +1370,13 @@ GENERATE_DOCSET = NO DOCSET_FEEDNAME = "Doxygen generated docs" +# This tag determines the URL of the docset feed. A documentation feed provides +# an umbrella under which multiple documentation sets from a single provider +# (such as a company or product suite) can be grouped. +# This tag requires that the tag GENERATE_DOCSET is set to YES. + +DOCSET_FEEDURL = + # This tag specifies a string that should uniquely identify the documentation # set bundle. This should be a reverse domain-name style string, e.g. # com.mycompany.MyDocSet. Doxygen will append .docset to the name. @@ -1446,8 +1402,12 @@ DOCSET_PUBLISHER_NAME = Publisher # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three # additional HTML index files: index.hhp, index.hhc, and index.hhk. The # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop -# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on -# Windows. +# on Windows. In the beginning of 2021 Microsoft took the original page, with +# a.o. the download links, offline the HTML help workshop was already many years +# in maintenance mode). You can download the HTML help workshop from the web +# archives at Installation executable (see: +# http://web.archive.org/web/20160201063255/http://download.microsoft.com/downlo +# ad/0/A/9/0A939EF6-E31C-430F-A3DF-DFAE7960D564/htmlhelp.exe). # # The HTML Help Workshop contains a compiler that can convert all HTML output # generated by doxygen into a single compiled HTML file (.chm). Compiled HTML @@ -1522,7 +1482,8 @@ QCH_FILE = # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help # Project output. For more information please see Qt Help Project / Namespace -# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). +# (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace). # The default value is: org.doxygen.Project. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1530,8 +1491,8 @@ QHP_NAMESPACE = org.doxygen.Project # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt # Help Project output. For more information please see Qt Help Project / Virtual -# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual- -# folders). +# Folders (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-folders). # The default value is: doc. # This tag requires that the tag GENERATE_QHP is set to YES. @@ -1539,16 +1500,16 @@ QHP_VIRTUAL_FOLDER = doc # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom # filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_NAME = # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the # custom filter to add. For more information please see Qt Help Project / Custom -# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom- -# filters). +# Filters (see: +# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-filters). # This tag requires that the tag GENERATE_QHP is set to YES. QHP_CUST_FILTER_ATTRS = @@ -1560,9 +1521,9 @@ QHP_CUST_FILTER_ATTRS = QHP_SECT_FILTER_ATTRS = -# The QHG_LOCATION tag can be used to specify the location of Qt's -# qhelpgenerator. If non-empty doxygen will try to run qhelpgenerator on the -# generated .qhp file. +# The QHG_LOCATION tag can be used to specify the location (absolute path +# including file name) of Qt's qhelpgenerator. If non-empty doxygen will try to +# run qhelpgenerator on the generated .qhp file. # This tag requires that the tag GENERATE_QHP is set to YES. QHG_LOCATION = @@ -1605,16 +1566,28 @@ DISABLE_INDEX = YES # to work a browser that supports JavaScript, DHTML, CSS and frames is required # (i.e. any modern browser). Windows users are probably better off using the # HTML help feature. Via custom style sheets (see HTML_EXTRA_STYLESHEET) one can -# further fine-tune the look of the index. As an example, the default style -# sheet generated by doxygen has an example that shows how to put an image at -# the root of the tree instead of the PROJECT_NAME. Since the tree basically has -# the same information as the tab index, you could consider setting -# DISABLE_INDEX to YES when enabling this option. +# further fine tune the look of the index (see "Fine-tuning the output"). As an +# example, the default style sheet generated by doxygen has an example that +# shows how to put an image at the root of the tree instead of the PROJECT_NAME. +# Since the tree basically has the same information as the tab index, you could +# consider setting DISABLE_INDEX to YES when enabling this option. # The default value is: NO. # This tag requires that the tag GENERATE_HTML is set to YES. GENERATE_TREEVIEW = YES +# When both GENERATE_TREEVIEW and DISABLE_INDEX are set to YES, then the +# FULL_SIDEBAR option determines if the side bar is limited to only the treeview +# area (value NO) or if it should extend to the full height of the window (value +# YES). Setting this to YES gives a layout similar to +# https://docs.readthedocs.io with more room for contents, but less room for the +# project logo, title, and description. If either GENERATE_TREEVIEW or +# DISABLE_INDEX is set to NO, this option has no effect. +# The default value is: NO. +# This tag requires that the tag GENERATE_HTML is set to YES. + +FULL_SIDEBAR = NO + # The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values that # doxygen will group on one line in the generated HTML documentation. # @@ -1687,11 +1660,29 @@ FORMULA_MACROFILE = USE_MATHJAX = YES +# With MATHJAX_VERSION it is possible to specify the MathJax version to be used. +# Note that the different versions of MathJax have different requirements with +# regards to the different settings, so it is possible that also other MathJax +# settings have to be changed when switching between the different MathJax +# versions. +# Possible values are: MathJax_2 and MathJax_3. +# The default value is: MathJax_2. +# This tag requires that the tag USE_MATHJAX is set to YES. + +MATHJAX_VERSION = MathJax_2 + # When MathJax is enabled you can set the default output format to be used for -# the MathJax output. See the MathJax site (see: -# http://docs.mathjax.org/en/latest/output.html) for more details. +# the MathJax output. For more details about the output format see MathJax +# version 2 (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) and MathJax version 3 +# (see: +# http://docs.mathjax.org/en/latest/web/components/output.html). # Possible values are: HTML-CSS (which is slower, but has the best -# compatibility), NativeMML (i.e. MathML) and SVG. +# compatibility. This is the name for Mathjax version 2, for MathJax version 3 +# this will be translated into chtml), NativeMML (i.e. MathML. Only supported +# for NathJax 2. For MathJax version 3 chtml will be used instead.), chtml (This +# is the name for Mathjax version 3, for MathJax version 2 this will be +# translated into HTML-CSS) and SVG. # The default value is: HTML-CSS. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1704,22 +1695,29 @@ MATHJAX_FORMAT = HTML-CSS # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax # Content Delivery Network so you can quickly see the result without installing # MathJax. However, it is strongly recommended to install a local copy of -# MathJax from https://www.mathjax.org before deployment. -# The default value is: https://cdn.jsdelivr.net/npm/mathjax@2. +# MathJax from https://www.mathjax.org before deployment. The default value is: +# - in case of MathJax version 2: https://cdn.jsdelivr.net/npm/mathjax@2 +# - in case of MathJax version 3: https://cdn.jsdelivr.net/npm/mathjax@3 # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_RELPATH = ./mathjax # The MATHJAX_EXTENSIONS tag can be used to specify one or more MathJax # extension names that should be enabled during MathJax rendering. For example +# for MathJax version 2 (see https://docs.mathjax.org/en/v2.7-latest/tex.html +# #tex-and-latex-extensions): # MATHJAX_EXTENSIONS = TeX/AMSmath TeX/AMSsymbols +# For example for MathJax version 3 (see +# http://docs.mathjax.org/en/latest/input/tex/extensions/index.html): +# MATHJAX_EXTENSIONS = ams # This tag requires that the tag USE_MATHJAX is set to YES. MATHJAX_EXTENSIONS = # The MATHJAX_CODEFILE tag can be used to specify a file with javascript pieces # of code that will be used on startup of the MathJax code. See the MathJax site -# (see: http://docs.mathjax.org/en/latest/output.html) for more details. For an +# (see: +# http://docs.mathjax.org/en/v2.7-latest/output.html) for more details. For an # example see the documentation. # This tag requires that the tag USE_MATHJAX is set to YES. @@ -1766,7 +1764,8 @@ SERVER_BASED_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). +# Xapian (see: +# https://xapian.org/). # # See the section "External Indexing and Searching" for details. # The default value is: NO. @@ -1779,8 +1778,9 @@ EXTERNAL_SEARCH = NO # # Doxygen ships with an example indexer (doxyindexer) and search engine # (doxysearch.cgi) which are based on the open source search engine library -# Xapian (see: https://xapian.org/). See the section "External Indexing and -# Searching" for details. +# Xapian (see: +# https://xapian.org/). See the section "External Indexing and Searching" for +# details. # This tag requires that the tag SEARCHENGINE is set to YES. SEARCHENGINE_URL = @@ -1889,29 +1889,31 @@ PAPER_TYPE = a4 EXTRA_PACKAGES = -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for the -# generated LaTeX document. The header should contain everything until the first -# chapter. If it is left blank doxygen will generate a standard header. See -# section "Doxygen usage" for information on how to let doxygen write the -# default header to a separate file. +# The LATEX_HEADER tag can be used to specify a user-defined LaTeX header for +# the generated LaTeX document. The header should contain everything until the +# first chapter. If it is left blank doxygen will generate a standard header. It +# is highly recommended to start with a default header using +# doxygen -w latex new_header.tex new_footer.tex new_stylesheet.sty +# and then modify the file new_header.tex. See also section "Doxygen usage" for +# information on how to generate the default header that doxygen normally uses. # -# Note: Only use a user-defined header if you know what you are doing! The -# following commands have a special meaning inside the header: $title, -# $datetime, $date, $doxygenversion, $projectname, $projectnumber, -# $projectbrief, $projectlogo. Doxygen will replace $title with the empty -# string, for the replacement values of the other commands the user is referred -# to HTML_HEADER. +# Note: Only use a user-defined header if you know what you are doing! +# Note: The header is subject to change so you typically have to regenerate the +# default header when upgrading to a newer version of doxygen. The following +# commands have a special meaning inside the header (and footer): For a +# description of the possible markers and block names see the documentation. # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_HEADER = -# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for the -# generated LaTeX document. The footer should contain everything after the last -# chapter. If it is left blank doxygen will generate a standard footer. See +# The LATEX_FOOTER tag can be used to specify a user-defined LaTeX footer for +# the generated LaTeX document. The footer should contain everything after the +# last chapter. If it is left blank doxygen will generate a standard footer. See # LATEX_HEADER for more information on how to generate a default footer and what -# special commands can be used inside the footer. -# -# Note: Only use a user-defined footer if you know what you are doing! +# special commands can be used inside the footer. See also section "Doxygen +# usage" for information on how to generate the default footer that doxygen +# normally uses. Note: Only use a user-defined footer if you know what you are +# doing! # This tag requires that the tag GENERATE_LATEX is set to YES. LATEX_FOOTER = @@ -1956,8 +1958,7 @@ USE_PDFLATEX = YES # If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \batchmode # command to the generated LaTeX files. This will instruct LaTeX to keep running -# if errors occur, instead of asking the user for help. This option is also used -# when generating formulas in HTML. +# if errors occur, instead of asking the user for help. # The default value is: NO. # This tag requires that the tag GENERATE_LATEX is set to YES. @@ -1970,16 +1971,6 @@ LATEX_BATCHMODE = NO LATEX_HIDE_INDICES = NO -# If the LATEX_SOURCE_CODE tag is set to YES then doxygen will include source -# code with syntax highlighting in the LaTeX output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_LATEX is set to YES. - -LATEX_SOURCE_CODE = NO - # The LATEX_BIB_STYLE tag can be used to specify the style to use for the # bibliography, e.g. plainnat, or ieeetr. See # https://en.wikipedia.org/wiki/BibTeX and \cite for more info. @@ -2060,16 +2051,6 @@ RTF_STYLESHEET_FILE = RTF_EXTENSIONS_FILE = -# If the RTF_SOURCE_CODE tag is set to YES then doxygen will include source code -# with syntax highlighting in the RTF output. -# -# Note that which sources are shown also depends on other settings such as -# SOURCE_BROWSER. -# The default value is: NO. -# This tag requires that the tag GENERATE_RTF is set to YES. - -RTF_SOURCE_CODE = NO - #--------------------------------------------------------------------------- # Configuration options related to the man page output #--------------------------------------------------------------------------- @@ -2166,15 +2147,6 @@ GENERATE_DOCBOOK = NO DOCBOOK_OUTPUT = docbook -# If the DOCBOOK_PROGRAMLISTING tag is set to YES, doxygen will include the -# program listings (including syntax highlighting and cross-referencing -# information) to the DOCBOOK output. Note that enabling this will significantly -# increase the size of the DOCBOOK output. -# The default value is: NO. -# This tag requires that the tag GENERATE_DOCBOOK is set to YES. - -DOCBOOK_PROGRAMLISTING = NO - #--------------------------------------------------------------------------- # Configuration options for the AutoGen Definitions output #--------------------------------------------------------------------------- @@ -2360,15 +2332,6 @@ EXTERNAL_PAGES = YES # Configuration options related to the dot tool #--------------------------------------------------------------------------- -# If the CLASS_DIAGRAMS tag is set to YES, doxygen will generate a class diagram -# (in HTML and LaTeX) for classes with base or super classes. Setting the tag to -# NO turns the diagrams off. Note that this option also works with HAVE_DOT -# disabled, but it is recommended to install and use dot, since it yields more -# powerful graphs. -# The default value is: YES. - -CLASS_DIAGRAMS = NO - # You can include diagrams made with dia in doxygen documentation. Doxygen will # then run dia to produce the diagram and insert it in the documentation. The # DIA_PATH tag allows you to specify the directory where the dia binary resides. @@ -2425,11 +2388,14 @@ DOT_FONTSIZE = 10 DOT_FONTPATH = -# If the CLASS_GRAPH tag is set to YES then doxygen will generate a graph for -# each documented class showing the direct and indirect inheritance relations. -# Setting this tag to YES will force the CLASS_DIAGRAMS tag to NO. +# If the CLASS_GRAPH tag is set to YES (or GRAPH) then doxygen will generate a +# graph for each documented class showing the direct and indirect inheritance +# relations. In case HAVE_DOT is set as well dot will be used to draw the graph, +# otherwise the built-in generator will be used. If the CLASS_GRAPH tag is set +# to TEXT the direct and indirect inheritance relations will be shown as texts / +# links. +# Possible values are: NO, YES, TEXT and GRAPH. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. CLASS_GRAPH = YES @@ -2466,10 +2432,32 @@ UML_LOOK = NO # but if the number exceeds 15, the total amount of fields shown is limited to # 10. # Minimum value: 0, maximum value: 100, default value: 10. -# This tag requires that the tag HAVE_DOT is set to YES. +# This tag requires that the tag UML_LOOK is set to YES. UML_LIMIT_NUM_FIELDS = 10 +# If the DOT_UML_DETAILS tag is set to NO, doxygen will show attributes and +# methods without types and arguments in the UML graphs. If the DOT_UML_DETAILS +# tag is set to YES, doxygen will add type and arguments for attributes and +# methods in the UML graphs. If the DOT_UML_DETAILS tag is set to NONE, doxygen +# will not generate fields with class member information in the UML graphs. The +# class diagrams will look similar to the default class diagrams but using UML +# notation for the relationships. +# Possible values are: NO, YES and NONE. +# The default value is: NO. +# This tag requires that the tag UML_LOOK is set to YES. + +DOT_UML_DETAILS = NO + +# The DOT_WRAP_THRESHOLD tag can be used to set the maximum number of characters +# to display on a single line. If the actual line length exceeds this threshold +# significantly it will wrapped across multiple lines. Some heuristics are apply +# to avoid ugly line breaks. +# Minimum value: 0, maximum value: 1000, default value: 17. +# This tag requires that the tag HAVE_DOT is set to YES. + +DOT_WRAP_THRESHOLD = 17 + # If the TEMPLATE_RELATIONS tag is set to YES then the inheritance and # collaboration graphs will show the relations between templates and their # instances. @@ -2536,6 +2524,13 @@ GRAPHICAL_HIERARCHY = YES DIRECTORY_GRAPH = YES +# The DIR_GRAPH_MAX_DEPTH tag can be used to limit the maximum number of levels +# of child directories generated in directory dependency graphs by dot. +# Minimum value: 1, maximum value: 25, default value: 1. +# This tag requires that the tag DIRECTORY_GRAPH is set to YES. + +DIR_GRAPH_MAX_DEPTH = 1 + # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images # generated by dot. For an explanation of the image formats see the section # output formats in the documentation of the dot tool (Graphviz (see: @@ -2659,9 +2654,11 @@ DOT_MULTI_TARGETS = NO GENERATE_LEGEND = YES -# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate dot +# If the DOT_CLEANUP tag is set to YES, doxygen will remove the intermediate # files that are used to generate the various graphs. +# +# Note: This setting is not only used for dot files but also for msc temporary +# files. # The default value is: YES. -# This tag requires that the tag HAVE_DOT is set to YES. DOT_CLEANUP = YES diff --git a/documentation/doxygen/Makefile b/documentation/doxygen/Makefile index 0656bacfa7c56..12bbd13e84159 100644 --- a/documentation/doxygen/Makefile +++ b/documentation/doxygen/Makefile @@ -15,8 +15,9 @@ export DOXYGEN_IMAGE_PATH := $(DOXYGEN_OUTPUT_DIRECTORY)/html export DOXYGEN_NOTEBOOK_PATH := $(DOXYGEN_OUTPUT_DIRECTORY)/notebooks export DOXYGEN_STRIP_PATH := $(shell cd $(PWD)/../..; pwd) export DOXYGEN_INCLUDE_PATH := $(shell find $(DOXYGEN_STRIP_PATH) -type d -name dictpch -prune -o -type d -name inc) +export DOXYGEN_PYZDOC_PATH := $(DOXYGEN_OUTPUT_DIRECTORY)/pyzdoc -PYZ_DIR = ../../bindings/pyroot/pythonizations/python/ROOT/pythonization +PYZ_DIR = $(DOXYGEN_SOURCE_DIRECTORY)/bindings/pyroot/pythonizations/python/ROOT/pythonization define MkDir +@[ -d $1 ] || mkdir -p $1 @@ -25,7 +26,7 @@ endef all: filter folders mathjax js images doxygen filter: - `root-config --cxx` -o filter filter.cxx -std=c++11 + `root-config --cxx` -o filter filter.cxx -std=c++14 mathjax: $(call MkDir,$(DOXYGEN_IMAGE_PATH)) @@ -40,18 +41,23 @@ images: cp images/* $(DOXYGEN_IMAGE_PATH) pyzdoc: - $(PYTHON_EXECUTABLE) extract_docstrings.py $(PYZ_DIR) - $(PYTHON_EXECUTABLE) print_roofit_pyz_doctrings.py > $(PYZ_DIR)/_roofit.pyzdoc + $(call MkDir,$(DOXYGEN_PYZDOC_PATH)) + $(PYTHON_EXECUTABLE) extract_docstrings.py $(PYZ_DIR) $(DOXYGEN_PYZDOC_PATH) + $(PYTHON_EXECUTABLE) print_roofit_pyz_doctrings.py > $(DOXYGEN_PYZDOC_PATH)/_roofit.pyzdoc doxygen: filter pyzdoc $(call MkDir,$(DOXYGEN_OUTPUT_DIRECTORY)) $(call MkDir,$(DOXYGEN_EXAMPLE_PATH)) $(call MkDir,$(DOXYGEN_NOTEBOOK_PATH)) ./makehtmlfooter.sh > htmlfooter.html + ./makeinput.sh doxygen ./listlibs.sh - rm -rf files c1* *.ps *.png *.svg *.pdf *.root *.xpm *.out *.dat *.dtd *.dot *.txt listofclass.sh + rm -rf files c1* *.ps *.eps *.png *.jpg *.tex *.svg *.pdf *.root *.xpm *.out *.dat *.dtd *.dot *.txt *.csv *.log *.rs + rm -rf listofclass.sh tmva* data* result* config* test* Roo* My* Freq* + rm -f Doxyfile_INPUT filter htmlfooter.html MDF.C pca.C + rm -f greek.gif hsumanim.gif mathsymb.gif parallelcoordtrans.gif rm -f $(DOXYGEN_NOTEBOOK_PATH)/*.root clean: - rm -r $(DOXYGEN_OUTPUT_DIRECTORY) filter htmlfooter.html dataset* $(PYZ_DIR)/*.pyzdoc + rm -rf $(DOXYGEN_OUTPUT_DIRECTORY) \ No newline at end of file diff --git a/documentation/doxygen/ROOT.css b/documentation/doxygen/ROOT.css index 492a38726517c..7f1e461b1356c 100644 --- a/documentation/doxygen/ROOT.css +++ b/documentation/doxygen/ROOT.css @@ -23,3 +23,9 @@ div.contents { background-color: #f9fafc; padding: 15px; } + +.legacybox { + border: 3px solid #879ecb; + background-color: #fdf06b; + padding: 15px; +} diff --git a/documentation/doxygen/extract_docstrings.py b/documentation/doxygen/extract_docstrings.py index d7dcd43f60534..02358d8cada6d 100755 --- a/documentation/doxygen/extract_docstrings.py +++ b/documentation/doxygen/extract_docstrings.py @@ -26,12 +26,12 @@ except ImportError: from scandir import scandir -if len(sys.argv) < 2: - print("Please provide the directory where documented .py files are.") +if len(sys.argv) < 3: + print("Please provide the directory where documented .py files are, and the output folder for the .pyzdoc files.") exit(1) pyz_dir = sys.argv[1] - +pyz_dir_out = sys.argv[2] def run_fast_scandir(dir, ext): # dir: str, ext: list subfolders, files = [], [] @@ -64,6 +64,12 @@ def run_fast_scandir(dir, ext): # dir: str, ext: list module = ast.parse(file_contents) ds = ast.get_docstring(module) if ds is not None: - with open(pyz_file_path + '.pyzdoc', 'w') as pyz_doc_file: + # The output file will sit in the $(DOXYGEN_PYZDOC_PATH)/pyzdoc folder. The output + # file name is build from the input file name which is extracted from pyz_file_path. + # The extracted file name has the extension ".py". We want the ouput file name has the + # extension ".pyzdoc". Therefore it is enough to concatenate the string "zdoc" at the + # end of the input file name to get the desired extension for the output file name. + pyz_filename_out = pyz_dir_out + '/' + pyz_file_path[pyz_file_path.rfind("_"):] + 'zdoc' + with open(pyz_filename_out, 'w') as pyz_doc_file: pyz_doc_file.write(ds) diff --git a/documentation/doxygen/filter.cxx b/documentation/doxygen/filter.cxx index d1881ac4b2d8f..c7611e79d682c 100644 --- a/documentation/doxygen/filter.cxx +++ b/documentation/doxygen/filter.cxx @@ -18,8 +18,8 @@ /// "png" is the default value. For example: `Begin_Macro(source, svg)` will show /// the code of the macro and the image will be is svg format. The "width" keyword /// can be added to define the width of the picture in pixel: "width=400" will -/// scale a picture to 400 pixel width. This allow to define large picture which -/// can then be scale done to have a better definition. +/// scale a picture to 400 pixel width. This allow to define large pictures which +/// can then be scaled down to have a better definition. /// /// ## In the ROOT tutorials /// diff --git a/documentation/doxygen/htmlheader.html b/documentation/doxygen/htmlheader.html index 40f4049137e3a..7a0f93961259a 100644 --- a/documentation/doxygen/htmlheader.html +++ b/documentation/doxygen/htmlheader.html @@ -1,5 +1,5 @@ - - + + diff --git a/documentation/doxygen/images/geom_booltree.png b/documentation/doxygen/images/geom_composite_shape001.png similarity index 100% rename from documentation/doxygen/images/geom_booltree.png rename to documentation/doxygen/images/geom_composite_shape001.png diff --git a/documentation/doxygen/images/geom_composite_shape002.png b/documentation/doxygen/images/geom_composite_shape002.png new file mode 100644 index 0000000000000..93e7c4df192d2 Binary files /dev/null and b/documentation/doxygen/images/geom_composite_shape002.png differ diff --git a/documentation/doxygen/images/geom_composite_shape003.png b/documentation/doxygen/images/geom_composite_shape003.png new file mode 100644 index 0000000000000..bf103b4325567 Binary files /dev/null and b/documentation/doxygen/images/geom_composite_shape003.png differ diff --git a/documentation/doxygen/images/geom_composite_shape004.png b/documentation/doxygen/images/geom_composite_shape004.png new file mode 100644 index 0000000000000..a69137f6cc539 Binary files /dev/null and b/documentation/doxygen/images/geom_composite_shape004.png differ diff --git a/documentation/doxygen/images/geom_primitive_shapes.png b/documentation/doxygen/images/geom_primitive_shapes.png new file mode 100644 index 0000000000000..b12684e04d2c1 Binary files /dev/null and b/documentation/doxygen/images/geom_primitive_shapes.png differ diff --git a/documentation/doxygen/images/geometry001.png b/documentation/doxygen/images/geometry001.png new file mode 100644 index 0000000000000..f97e436321a5f Binary files /dev/null and b/documentation/doxygen/images/geometry001.png differ diff --git a/documentation/doxygen/images/geometry002.jpg b/documentation/doxygen/images/geometry002.jpg new file mode 100644 index 0000000000000..444627e8bc3d9 Binary files /dev/null and b/documentation/doxygen/images/geometry002.jpg differ diff --git a/documentation/doxygen/images/geometry003.png b/documentation/doxygen/images/geometry003.png new file mode 100644 index 0000000000000..ec68a75cd35e6 Binary files /dev/null and b/documentation/doxygen/images/geometry003.png differ diff --git a/documentation/doxygen/images/geometry004.png b/documentation/doxygen/images/geometry004.png new file mode 100644 index 0000000000000..27f81d205c353 Binary files /dev/null and b/documentation/doxygen/images/geometry004.png differ diff --git a/documentation/doxygen/images/geometry005.png b/documentation/doxygen/images/geometry005.png new file mode 100644 index 0000000000000..9e5fbad43f346 Binary files /dev/null and b/documentation/doxygen/images/geometry005.png differ diff --git a/documentation/doxygen/images/geometry006.png b/documentation/doxygen/images/geometry006.png new file mode 100644 index 0000000000000..cc9bcb9d7ac72 Binary files /dev/null and b/documentation/doxygen/images/geometry006.png differ diff --git a/documentation/doxygen/images/geometry007.png b/documentation/doxygen/images/geometry007.png new file mode 100644 index 0000000000000..57649cc9ed250 Binary files /dev/null and b/documentation/doxygen/images/geometry007.png differ diff --git a/documentation/doxygen/images/geometry008.png b/documentation/doxygen/images/geometry008.png new file mode 100644 index 0000000000000..27f36df5b324e Binary files /dev/null and b/documentation/doxygen/images/geometry008.png differ diff --git a/documentation/doxygen/images/geometry009.png b/documentation/doxygen/images/geometry009.png new file mode 100644 index 0000000000000..cc5b2576080c0 Binary files /dev/null and b/documentation/doxygen/images/geometry009.png differ diff --git a/documentation/doxygen/images/geometry010.png b/documentation/doxygen/images/geometry010.png new file mode 100644 index 0000000000000..50656dff87722 Binary files /dev/null and b/documentation/doxygen/images/geometry010.png differ diff --git a/documentation/doxygen/images/geometry011.png b/documentation/doxygen/images/geometry011.png new file mode 100644 index 0000000000000..7bced014b2224 Binary files /dev/null and b/documentation/doxygen/images/geometry011.png differ diff --git a/documentation/doxygen/images/geometry012.png b/documentation/doxygen/images/geometry012.png new file mode 100644 index 0000000000000..f12b3fede8ed1 Binary files /dev/null and b/documentation/doxygen/images/geometry012.png differ diff --git a/documentation/doxygen/images/geometry013.jpg b/documentation/doxygen/images/geometry013.jpg new file mode 100644 index 0000000000000..bca84d5af59e0 Binary files /dev/null and b/documentation/doxygen/images/geometry013.jpg differ diff --git a/documentation/doxygen/images/geometry014.png b/documentation/doxygen/images/geometry014.png new file mode 100644 index 0000000000000..8e6452870d47b Binary files /dev/null and b/documentation/doxygen/images/geometry014.png differ diff --git a/documentation/doxygen/images/geometry015.png b/documentation/doxygen/images/geometry015.png new file mode 100644 index 0000000000000..b0a77ecba3b3c Binary files /dev/null and b/documentation/doxygen/images/geometry015.png differ diff --git a/documentation/doxygen/images/geometry016.png b/documentation/doxygen/images/geometry016.png new file mode 100644 index 0000000000000..8fee1502e46e3 Binary files /dev/null and b/documentation/doxygen/images/geometry016.png differ diff --git a/documentation/doxygen/images/geometry017.png b/documentation/doxygen/images/geometry017.png new file mode 100644 index 0000000000000..f9dd0277c90be Binary files /dev/null and b/documentation/doxygen/images/geometry017.png differ diff --git a/documentation/doxygen/images/geometry018.png b/documentation/doxygen/images/geometry018.png new file mode 100644 index 0000000000000..11b361bd7412e Binary files /dev/null and b/documentation/doxygen/images/geometry018.png differ diff --git a/documentation/doxygen/images/geometry019.png b/documentation/doxygen/images/geometry019.png new file mode 100644 index 0000000000000..10cd4148195a8 Binary files /dev/null and b/documentation/doxygen/images/geometry019.png differ diff --git a/documentation/doxygen/images/geometry020.png b/documentation/doxygen/images/geometry020.png new file mode 100644 index 0000000000000..a9a36a8d83009 Binary files /dev/null and b/documentation/doxygen/images/geometry020.png differ diff --git a/documentation/doxygen/images/geometry021.png b/documentation/doxygen/images/geometry021.png new file mode 100644 index 0000000000000..0a1f2eaec0fd9 Binary files /dev/null and b/documentation/doxygen/images/geometry021.png differ diff --git a/documentation/doxygen/images/geometry022.jpg b/documentation/doxygen/images/geometry022.jpg new file mode 100644 index 0000000000000..be328e470dea4 Binary files /dev/null and b/documentation/doxygen/images/geometry022.jpg differ diff --git a/documentation/doxygen/images/geometry023.jpg b/documentation/doxygen/images/geometry023.jpg new file mode 100644 index 0000000000000..5d9f857ce6b3b Binary files /dev/null and b/documentation/doxygen/images/geometry023.jpg differ diff --git a/documentation/doxygen/images/geometry024.jpg b/documentation/doxygen/images/geometry024.jpg new file mode 100644 index 0000000000000..6a7373d7e3e8c Binary files /dev/null and b/documentation/doxygen/images/geometry024.jpg differ diff --git a/documentation/doxygen/images/geometry025.jpg b/documentation/doxygen/images/geometry025.jpg new file mode 100644 index 0000000000000..0c9896c8ddfdd Binary files /dev/null and b/documentation/doxygen/images/geometry025.jpg differ diff --git a/documentation/doxygen/images/splot_fisher-bkg-sPlot.png b/documentation/doxygen/images/splot_fisher-bkg-sPlot.png new file mode 100644 index 0000000000000..afa452c2ee774 Binary files /dev/null and b/documentation/doxygen/images/splot_fisher-bkg-sPlot.png differ diff --git a/documentation/doxygen/makehtmlfooter.sh b/documentation/doxygen/makehtmlfooter.sh index e4e693656d1b5..7a15d13de48a7 100755 --- a/documentation/doxygen/makehtmlfooter.sh +++ b/documentation/doxygen/makehtmlfooter.sh @@ -2,7 +2,7 @@ # Generates the HTML footer -echo '' +echo '' echo '' echo '' echo '