From 99c8950e2f9878b068b879872cab2714aed46f5f Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 29 Aug 2022 14:12:32 +0200 Subject: [PATCH 01/35] Make SourceInterface cloneable --- .../SEFramework/Property/PropertyHolder.h | 9 ++++++--- SEFramework/SEFramework/Source/SimpleSource.h | 9 ++++++++- .../SEFramework/Source/SimpleSourceGroup.h | 5 ++++- .../SEFramework/Source/SourceGroupInterface.h | 11 +++++++++-- .../Source/SourceGroupWithOnDemandProperties.h | 9 +++++++-- .../SEFramework/Source/SourceInterface.h | 5 ++++- .../Source/SourceWithOnDemandProperties.h | 9 ++++++--- SEFramework/src/lib/Property/PropertyHolder.cpp | 7 ++++++- SEFramework/src/lib/Source/EntangledSource.cpp | 7 ++++++- .../src/lib/Source/SimpleSourceGroup.cpp | 17 +++++++++++++++-- .../SourceGroupWithOnDemandProperties.cpp | 16 +++++++++++++++- .../lib/Source/SourceWithOnDemandProperties.cpp | 13 +++++++++++-- .../tests/src/Source/SourceInterface_test.cpp | 7 ++++++- 13 files changed, 103 insertions(+), 21 deletions(-) diff --git a/SEFramework/SEFramework/Property/PropertyHolder.h b/SEFramework/SEFramework/Property/PropertyHolder.h index de95a8288..81b8f412f 100644 --- a/SEFramework/SEFramework/Property/PropertyHolder.h +++ b/SEFramework/SEFramework/Property/PropertyHolder.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -64,12 +65,14 @@ class PropertyHolder { /// Returns true if the property is set bool isPropertySet(const PropertyId& property_id) const; - + void clear(); + void update(const PropertyHolder& other); + private: - std::unordered_map> m_properties; + std::unordered_map> m_properties; }; /* End of ObjectWithProperties class */ diff --git a/SEFramework/SEFramework/Source/SimpleSource.h b/SEFramework/SEFramework/Source/SimpleSource.h index 97e3d09be..e3e9d8a7d 100644 --- a/SEFramework/SEFramework/Source/SimpleSource.h +++ b/SEFramework/SEFramework/Source/SimpleSource.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -54,6 +55,12 @@ class SimpleSource : public SourceInterface { /// Constructor SimpleSource() {} + std::unique_ptr clone() const override { + auto cloned = std::make_unique(); + cloned->m_property_holder.update(m_property_holder); + return std::move(cloned); + } + // Note : Because the get/setProperty() methods of the SourceInterface are // templated, the overrides of the non-templated versions will hide them. For // this reason it is necessary to re-introduce the templated methods, which is diff --git a/SEFramework/SEFramework/Source/SimpleSourceGroup.h b/SEFramework/SEFramework/Source/SimpleSourceGroup.h index fc9130eb4..e152ac73c 100644 --- a/SEFramework/SEFramework/Source/SimpleSourceGroup.h +++ b/SEFramework/SEFramework/Source/SimpleSourceGroup.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -62,6 +63,8 @@ class SimpleSourceGroup : public SourceGroupInterface { void merge(SourceGroupInterface&& other) override; + std::unique_ptr clone() const override; + using SourceInterface::getProperty; using SourceInterface::setProperty; diff --git a/SEFramework/SEFramework/Source/SourceGroupInterface.h b/SEFramework/SEFramework/Source/SourceGroupInterface.h index 1af5f902d..199e59903 100644 --- a/SEFramework/SEFramework/Source/SourceGroupInterface.h +++ b/SEFramework/SEFramework/Source/SourceGroupInterface.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -23,6 +24,7 @@ #define _SEFRAMEWORK_SOURCEGROUPINTERFACE_H #include "SEFramework/Source/SourceInterface.h" +#include "AlexandriaKernel/memory_tools.h" namespace SourceXtractor { @@ -34,7 +36,7 @@ namespace SourceXtractor { * */ -class SourceGroupInterface : protected SourceInterface { +class SourceGroupInterface : public SourceInterface { template using CollectionType = typename std::iterator_traits::value_type; @@ -58,6 +60,10 @@ class SourceGroupInterface : protected SourceInterface { m_source->setProperty(std::move(property), property_id); } + std::unique_ptr clone() const override { + return Euclid::make_unique(m_source->clone()); + } + bool operator<(const SourceWrapper& other) const { return this->m_source < other.m_source; } @@ -107,6 +113,7 @@ class SourceGroupInterface : protected SourceInterface { using SourceInterface::getProperty; using SourceInterface::setProperty; using SourceInterface::setIndexedProperty; + using SourceInterface::clone; }; // end of SourceGroupInterface class diff --git a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h index 9b063946f..f31d7e45f 100644 --- a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -70,6 +71,8 @@ class SourceGroupWithOnDemandProperties : public SourceGroupInterface { unsigned int size() const override; + std::unique_ptr clone() const override; + using SourceInterface::getProperty; using SourceInterface::setProperty; @@ -104,6 +107,8 @@ class SourceGroupWithOnDemandProperties::EntangledSource : public SourceInterfac void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + std::unique_ptr clone() const override; + bool operator<(const EntangledSource& other) const; private: @@ -114,7 +119,7 @@ class SourceGroupWithOnDemandProperties::EntangledSource : public SourceInterfac friend void SourceGroupWithOnDemandProperties::clearGroupProperties(); friend void SourceGroupWithOnDemandProperties::merge(SourceGroupInterface&&); - + friend std::unique_ptr SourceGroupWithOnDemandProperties::clone() const; }; } /* namespace SourceXtractor */ diff --git a/SEFramework/SEFramework/Source/SourceInterface.h b/SEFramework/SEFramework/Source/SourceInterface.h index 3ecb1eeea..8aa12d603 100644 --- a/SEFramework/SEFramework/Source/SourceInterface.h +++ b/SEFramework/SEFramework/Source/SourceInterface.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -78,6 +79,8 @@ class SourceInterface { virtual const Property& getProperty(const PropertyId& property_id) const = 0; virtual void setProperty(std::unique_ptr property, const PropertyId& property_id) = 0; + virtual std::unique_ptr clone() const = 0; + }; /* End of SourceInterface class */ } /* namespace SourceXtractor */ diff --git a/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h index aef4eb8ea..71d21861d 100644 --- a/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -50,8 +51,6 @@ class SourceWithOnDemandProperties : public SourceInterface { virtual ~SourceWithOnDemandProperties() = default; // removes copy/move constructors and assignment operators - - SourceWithOnDemandProperties(const SourceWithOnDemandProperties&) = delete; SourceWithOnDemandProperties& operator=(const SourceWithOnDemandProperties&) = delete; SourceWithOnDemandProperties(SourceWithOnDemandProperties&&) = delete; SourceWithOnDemandProperties& operator=(SourceWithOnDemandProperties&&) = delete; @@ -65,6 +64,8 @@ class SourceWithOnDemandProperties : public SourceInterface { // done by the using statements below. using SourceInterface::getProperty; using SourceInterface::setProperty; + + std::unique_ptr clone() const override; protected: @@ -73,6 +74,8 @@ class SourceWithOnDemandProperties : public SourceInterface { void setProperty(std::unique_ptr property, const PropertyId& property_id) override; private: + SourceWithOnDemandProperties(const SourceWithOnDemandProperties& other); + std::shared_ptr m_task_provider; PropertyHolder m_property_holder; }; /* End of Source class */ diff --git a/SEFramework/src/lib/Property/PropertyHolder.cpp b/SEFramework/src/lib/Property/PropertyHolder.cpp index 8436898ca..a2d26710d 100644 --- a/SEFramework/src/lib/Property/PropertyHolder.cpp +++ b/SEFramework/src/lib/Property/PropertyHolder.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -49,4 +50,8 @@ void PropertyHolder::clear() { m_properties.clear(); } +void PropertyHolder::update(const SourceXtractor::PropertyHolder& other) { + std::copy(other.m_properties.begin(), other.m_properties.end(), std::inserter(m_properties, m_properties.begin())); +} + } // SEFramework namespace diff --git a/SEFramework/src/lib/Source/EntangledSource.cpp b/SEFramework/src/lib/Source/EntangledSource.cpp index 32db767e0..d7d13c0af 100644 --- a/SEFramework/src/lib/Source/EntangledSource.cpp +++ b/SEFramework/src/lib/Source/EntangledSource.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -80,6 +81,10 @@ void SourceGroupWithOnDemandProperties::EntangledSource::setProperty(std::unique m_property_holder.setProperty(std::move(property), property_id); } +std::unique_ptr SourceGroupWithOnDemandProperties::EntangledSource::clone() const { + throw Elements::Exception("Can not clone an entangled source"); +} + bool SourceGroupWithOnDemandProperties::EntangledSource::operator<(const EntangledSource& other) const { return this->m_source < other.m_source; } diff --git a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp index 3884faed7..febc01c17 100644 --- a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp +++ b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -20,6 +21,9 @@ */ #include "SEFramework/Source/SimpleSourceGroup.h" +#include "AlexandriaKernel/memory_tools.h" + +using Euclid::make_unique; namespace SourceXtractor { @@ -77,4 +81,13 @@ unsigned int SimpleSourceGroup::size() const { return m_sources.size(); } -} // SourceXtractor namespace +std::unique_ptr SimpleSourceGroup::clone() const { + auto cloned = make_unique(); + for (const auto& src : m_sources) { + cloned->addSource(src.getRef().clone()); + } + cloned->m_property_holder.update(m_property_holder); + return cloned; +} + +} // namespace SourceXtractor diff --git a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp index 23591d976..b382aff5e 100644 --- a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -22,6 +23,9 @@ #include "SEFramework/Source/SourceGroupWithOnDemandProperties.h" #include "SEFramework/Task/GroupTask.h" +#include "AlexandriaKernel/memory_tools.h" + +using Euclid::make_unique; namespace SourceXtractor { @@ -109,6 +113,16 @@ unsigned int SourceGroupWithOnDemandProperties::size() const { return m_sources.size(); } +std::unique_ptr SourceGroupWithOnDemandProperties::clone() const { + auto cloned = make_unique(m_task_provider); + for (const auto& source : m_sources) { + auto& entangled_source = dynamic_cast(source.getRef()); + cloned->m_sources.emplace_back(Euclid::make_unique(entangled_source.m_source, *cloned)); + } + cloned->m_property_holder.update(this->m_property_holder); + return cloned; +} + } // SourceXtractor namespace diff --git a/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp index 16f9b7fa6..e12ad8b19 100644 --- a/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -23,7 +24,6 @@ #include "SEFramework/Task/TaskProvider.h" #include "SEFramework/Task/SourceTask.h" #include "SEFramework/Property/PropertyNotFoundException.h" - #include "SEFramework/Source/SourceWithOnDemandProperties.h" namespace SourceXtractor { @@ -32,6 +32,11 @@ SourceWithOnDemandProperties::SourceWithOnDemandProperties(std::shared_ptr propert m_property_holder.setProperty(std::move(property), property_id); } +std::unique_ptr SourceWithOnDemandProperties::clone() const { + return std::unique_ptr(new SourceWithOnDemandProperties(*this)); +} + } // SEFramework namespace diff --git a/SEFramework/tests/src/Source/SourceInterface_test.cpp b/SEFramework/tests/src/Source/SourceInterface_test.cpp index 34700b697..57dbe00df 100644 --- a/SEFramework/tests/src/Source/SourceInterface_test.cpp +++ b/SEFramework/tests/src/Source/SourceInterface_test.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -52,6 +53,10 @@ class MockSourceInterface : public SourceInterface { MOCK_CONST_METHOD1(getProperty, Property& (const PropertyId& property_id)); void setProperty(std::unique_ptr, const PropertyId& ) {} + + std::unique_ptr clone() const override { + return nullptr; + } }; //----------------------------------------------------------------------------- From c2219333f683131ab96f87e5b210b68414ef1f10 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 2 Sep 2022 15:00:42 +0200 Subject: [PATCH 02/35] EntangledSource owns the wrapped source --- .../Source/SourceGroupWithOnDemandProperties.h | 4 ++-- SEFramework/src/lib/Source/EntangledSource.cpp | 8 ++++---- .../src/lib/Source/SourceGroupWithOnDemandProperties.cpp | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h index f31d7e45f..d968c7290 100644 --- a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h @@ -99,7 +99,7 @@ class SourceGroupWithOnDemandProperties::EntangledSource : public SourceInterfac public: - EntangledSource(std::shared_ptr source, SourceGroupWithOnDemandProperties& group); + EntangledSource(std::unique_ptr source, SourceGroupWithOnDemandProperties& group); virtual ~EntangledSource() = default; @@ -114,7 +114,7 @@ class SourceGroupWithOnDemandProperties::EntangledSource : public SourceInterfac private: PropertyHolder m_property_holder; - std::shared_ptr m_source; + std::unique_ptr m_source; SourceGroupWithOnDemandProperties& m_group; friend void SourceGroupWithOnDemandProperties::clearGroupProperties(); diff --git a/SEFramework/src/lib/Source/EntangledSource.cpp b/SEFramework/src/lib/Source/EntangledSource.cpp index d7d13c0af..1da73a339 100644 --- a/SEFramework/src/lib/Source/EntangledSource.cpp +++ b/SEFramework/src/lib/Source/EntangledSource.cpp @@ -25,16 +25,16 @@ namespace SourceXtractor { -SourceGroupWithOnDemandProperties::EntangledSource::EntangledSource(std::shared_ptr source, SourceGroupWithOnDemandProperties& group) - : m_source(source), m_group(group) { +SourceGroupWithOnDemandProperties::EntangledSource::EntangledSource(std::unique_ptr source, SourceGroupWithOnDemandProperties& group) + : m_source(std::move(source)), m_group(group) { // Normally, it should not be possible that the given source is of type // EntangledSource, because the entangled sources of a group can only be // accessed via the iterator as references. Nevertheless, to be sure that // future changes will not change the behavior, we do a check to the given // source and if it is an EntangledSource we use its encapsulated source instead. - auto entangled_ptr = std::dynamic_pointer_cast(m_source); + auto entangled_ptr = dynamic_cast(m_source.get()); if (entangled_ptr != nullptr) { - m_source = entangled_ptr->m_source; + m_source = std::move(entangled_ptr->m_source); } } diff --git a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp index b382aff5e..5fc1a0bd0 100644 --- a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp @@ -117,7 +117,7 @@ std::unique_ptr SourceGroupWithOnDemandProperties::clone() cons auto cloned = make_unique(m_task_provider); for (const auto& source : m_sources) { auto& entangled_source = dynamic_cast(source.getRef()); - cloned->m_sources.emplace_back(Euclid::make_unique(entangled_source.m_source, *cloned)); + cloned->m_sources.emplace_back(Euclid::make_unique(entangled_source.m_source->clone(), *cloned)); } cloned->m_property_holder.update(this->m_property_holder); return cloned; From c0e72953f46f3c6210c4a167558cf054ffd1c647 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 16 Sep 2022 13:10:17 +0200 Subject: [PATCH 03/35] Fix build on centos7 --- SEFramework/SEFramework/Source/SimpleSource.h | 5 +++-- SEFramework/src/lib/Source/SimpleSourceGroup.cpp | 2 +- .../src/lib/Source/SourceGroupWithOnDemandProperties.cpp | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/SEFramework/SEFramework/Source/SimpleSource.h b/SEFramework/SEFramework/Source/SimpleSource.h index e3e9d8a7d..c42103004 100644 --- a/SEFramework/SEFramework/Source/SimpleSource.h +++ b/SEFramework/SEFramework/Source/SimpleSource.h @@ -25,8 +25,9 @@ #ifndef _SEFRAMEWORK_SOURCE_SIMPLESOURCE_H_ #define _SEFRAMEWORK_SOURCE_SIMPLESOURCE_H_ -#include "SEFramework/Source/SourceInterface.h" +#include "AlexandriaKernel/memory_tools.h" #include "SEFramework/Property/PropertyHolder.h" +#include "SEFramework/Source/SourceInterface.h" namespace SourceXtractor { @@ -56,7 +57,7 @@ class SimpleSource : public SourceInterface { SimpleSource() {} std::unique_ptr clone() const override { - auto cloned = std::make_unique(); + auto cloned = Euclid::make_unique(); cloned->m_property_holder.update(m_property_holder); return std::move(cloned); } diff --git a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp index febc01c17..126cc598d 100644 --- a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp +++ b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp @@ -87,7 +87,7 @@ std::unique_ptr SimpleSourceGroup::clone() const { cloned->addSource(src.getRef().clone()); } cloned->m_property_holder.update(m_property_holder); - return cloned; + return std::unique_ptr(std::move(cloned)); } } // namespace SourceXtractor diff --git a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp index 5fc1a0bd0..0ebc3bb1f 100644 --- a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp @@ -120,7 +120,7 @@ std::unique_ptr SourceGroupWithOnDemandProperties::clone() cons cloned->m_sources.emplace_back(Euclid::make_unique(entangled_source.m_source->clone(), *cloned)); } cloned->m_property_holder.update(this->m_property_holder); - return cloned; + return std::unique_ptr(std::move(cloned)); } } // SourceXtractor namespace From e1314ed8be8d7b02da8a86aef4db417d83ae3061 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 5 Sep 2022 12:07:13 +0200 Subject: [PATCH 04/35] Add method to iterate set properties --- .../SEFramework/Property/PropertyHolder.h | 12 +++++++++++- SEFramework/SEFramework/Source/SimpleSource.h | 8 +++++++- .../SEFramework/Source/SimpleSourceGroup.h | 4 +++- .../SEFramework/Source/SourceGroupInterface.h | 6 +++++- .../Source/SourceGroupWithOnDemandProperties.h | 8 ++++++-- SEFramework/SEFramework/Source/SourceInterface.h | 8 +++++++- .../Source/SourceWithOnDemandProperties.h | 4 +++- SEFramework/src/lib/Property/PropertyHolder.cpp | 2 +- SEFramework/src/lib/Source/EntangledSource.cpp | 3 ++- SEFramework/src/lib/Source/SimpleSourceGroup.cpp | 9 ++++++++- .../Source/SourceGroupWithOnDemandProperties.cpp | 16 +++++++++++++++- .../lib/Source/SourceWithOnDemandProperties.cpp | 7 ++++++- .../tests/src/Source/SourceInterface_test.cpp | 4 +++- 13 files changed, 77 insertions(+), 14 deletions(-) diff --git a/SEFramework/SEFramework/Property/PropertyHolder.h b/SEFramework/SEFramework/Property/PropertyHolder.h index 81b8f412f..133bc882e 100644 --- a/SEFramework/SEFramework/Property/PropertyHolder.h +++ b/SEFramework/SEFramework/Property/PropertyHolder.h @@ -45,6 +45,8 @@ class PropertyHolder { public: + using const_iterator = std::unordered_map>::const_iterator; + /// Destructor virtual ~PropertyHolder() = default; @@ -61,7 +63,7 @@ class PropertyHolder { const Property& getProperty(const PropertyId& property_id) const; /// Sets a property, overwriting it if necessary - void setProperty(std::unique_ptr property, const PropertyId& property_id); + void setProperty(std::shared_ptr property, const PropertyId& property_id); /// Returns true if the property is set bool isPropertySet(const PropertyId& property_id) const; @@ -70,6 +72,14 @@ class PropertyHolder { void update(const PropertyHolder& other); + const_iterator begin() const { + return m_properties.begin(); + } + + const_iterator end() const { + return m_properties.end(); + } + private: std::unordered_map> m_properties; diff --git a/SEFramework/SEFramework/Source/SimpleSource.h b/SEFramework/SEFramework/Source/SimpleSource.h index c42103004..5becd205f 100644 --- a/SEFramework/SEFramework/Source/SimpleSource.h +++ b/SEFramework/SEFramework/Source/SimpleSource.h @@ -62,6 +62,12 @@ class SimpleSource : public SourceInterface { return std::move(cloned); } + void visitProperties(const PropertyVisitor& visitor) override { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); + } + // Note : Because the get/setProperty() methods of the SourceInterface are // templated, the overrides of the non-templated versions will hide them. For // this reason it is necessary to re-introduce the templated methods, which is @@ -76,7 +82,7 @@ class SimpleSource : public SourceInterface { return m_property_holder.getProperty(property_id); } - void setProperty(std::unique_ptr property, const PropertyId& property_id) override { + void setProperty(std::shared_ptr property, const PropertyId& property_id) override { m_property_holder.setProperty(std::move(property), property_id); } diff --git a/SEFramework/SEFramework/Source/SimpleSourceGroup.h b/SEFramework/SEFramework/Source/SimpleSourceGroup.h index e152ac73c..2b31fe8cf 100644 --- a/SEFramework/SEFramework/Source/SimpleSourceGroup.h +++ b/SEFramework/SEFramework/Source/SimpleSourceGroup.h @@ -65,6 +65,8 @@ class SimpleSourceGroup : public SourceGroupInterface { std::unique_ptr clone() const override; + void visitProperties(const PropertyVisitor& visitor) override; + using SourceInterface::getProperty; using SourceInterface::setProperty; @@ -72,7 +74,7 @@ class SimpleSourceGroup : public SourceGroupInterface { const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; private: diff --git a/SEFramework/SEFramework/Source/SourceGroupInterface.h b/SEFramework/SEFramework/Source/SourceGroupInterface.h index 199e59903..b08866101 100644 --- a/SEFramework/SEFramework/Source/SourceGroupInterface.h +++ b/SEFramework/SEFramework/Source/SourceGroupInterface.h @@ -56,10 +56,14 @@ class SourceGroupInterface : public SourceInterface { return m_source->getProperty(property_id); } - void setProperty(std::unique_ptr property, const PropertyId& property_id) override { + void setProperty(std::shared_ptr property, const PropertyId& property_id) override { m_source->setProperty(std::move(property), property_id); } + void visitProperties(const PropertyVisitor& visitor) override { + m_source->visitProperties(visitor); + } + std::unique_ptr clone() const override { return Euclid::make_unique(m_source->clone()); } diff --git a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h index d968c7290..0feba9d1c 100644 --- a/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceGroupWithOnDemandProperties.h @@ -71,6 +71,8 @@ class SourceGroupWithOnDemandProperties : public SourceGroupInterface { unsigned int size() const override; + void visitProperties(const PropertyVisitor& visitor) override; + std::unique_ptr clone() const override; using SourceInterface::getProperty; @@ -80,7 +82,7 @@ class SourceGroupWithOnDemandProperties : public SourceGroupInterface { const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; private: @@ -105,7 +107,9 @@ class SourceGroupWithOnDemandProperties::EntangledSource : public SourceInterfac const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; + + void visitProperties(const PropertyVisitor& visitor) override; std::unique_ptr clone() const override; diff --git a/SEFramework/SEFramework/Source/SourceInterface.h b/SEFramework/SEFramework/Source/SourceInterface.h index 8aa12d603..9c047a8fc 100644 --- a/SEFramework/SEFramework/Source/SourceInterface.h +++ b/SEFramework/SEFramework/Source/SourceInterface.h @@ -47,6 +47,7 @@ namespace SourceXtractor { class SourceInterface { public: + using PropertyVisitor = std::function&)>; /** * @brief Destructor @@ -74,10 +75,15 @@ class SourceInterface { setIndexedProperty(0, std::forward(args)...); } + /// Call PropertyVisitor once per *set* property. On demand properties are ignored if not yet computed. + /// Concrete implementations must call the visitor from lower to higher preference (i.e. group properties before + /// individual source properties), so in case of conflict the higher priority property can take precedence. + virtual void visitProperties(const PropertyVisitor&) = 0; + /// Returns a reference to the requested property. The property may be computed if needed /// Throws a PropertyNotFoundException if the property cannot be provided. virtual const Property& getProperty(const PropertyId& property_id) const = 0; - virtual void setProperty(std::unique_ptr property, const PropertyId& property_id) = 0; + virtual void setProperty(std::shared_ptr property, const PropertyId& property_id) = 0; virtual std::unique_ptr clone() const = 0; diff --git a/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h b/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h index 71d21861d..547786080 100644 --- a/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h +++ b/SEFramework/SEFramework/Source/SourceWithOnDemandProperties.h @@ -58,6 +58,8 @@ class SourceWithOnDemandProperties : public SourceInterface { /// Constructor explicit SourceWithOnDemandProperties(std::shared_ptr task_provider); + void visitProperties(const PropertyVisitor& visitor) override; + // Note : Because the get/setProperty() methods of the SourceInterface are // templated, the overrides of the non-templated versions will hide them. For // this reason it is necessary to re-introduce the templated methods, which is @@ -71,7 +73,7 @@ class SourceWithOnDemandProperties : public SourceInterface { // Implementation of SourceInterface const Property& getProperty(const PropertyId& property_id) const override; - void setProperty(std::unique_ptr property, const PropertyId& property_id) override; + void setProperty(std::shared_ptr property, const PropertyId& property_id) override; private: SourceWithOnDemandProperties(const SourceWithOnDemandProperties& other); diff --git a/SEFramework/src/lib/Property/PropertyHolder.cpp b/SEFramework/src/lib/Property/PropertyHolder.cpp index a2d26710d..add189bd8 100644 --- a/SEFramework/src/lib/Property/PropertyHolder.cpp +++ b/SEFramework/src/lib/Property/PropertyHolder.cpp @@ -38,7 +38,7 @@ const Property& PropertyHolder::getProperty(const PropertyId& property_id) const } } -void PropertyHolder::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void PropertyHolder::setProperty(std::shared_ptr property, const PropertyId& property_id) { m_properties[property_id] = std::move(property); } diff --git a/SEFramework/src/lib/Source/EntangledSource.cpp b/SEFramework/src/lib/Source/EntangledSource.cpp index 1da73a339..2b4af1ac1 100644 --- a/SEFramework/src/lib/Source/EntangledSource.cpp +++ b/SEFramework/src/lib/Source/EntangledSource.cpp @@ -77,7 +77,8 @@ const Property& SourceGroupWithOnDemandProperties::EntangledSource::getProperty( } // end of getProperty() -void SourceGroupWithOnDemandProperties::EntangledSource::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SourceGroupWithOnDemandProperties::EntangledSource::setProperty(std::shared_ptr property, + const PropertyId& property_id) { m_property_holder.setProperty(std::move(property), property_id); } diff --git a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp index 126cc598d..221d66162 100644 --- a/SEFramework/src/lib/Source/SimpleSourceGroup.cpp +++ b/SEFramework/src/lib/Source/SimpleSourceGroup.cpp @@ -22,6 +22,7 @@ #include "SEFramework/Source/SimpleSourceGroup.h" #include "AlexandriaKernel/memory_tools.h" +#include using Euclid::make_unique; @@ -73,7 +74,7 @@ const Property& SimpleSourceGroup::getProperty(const PropertyId& property_id) co return m_property_holder.getProperty(property_id); } -void SimpleSourceGroup::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SimpleSourceGroup::setProperty(std::shared_ptr property, const PropertyId& property_id) { m_property_holder.setProperty(std::move(property), property_id); } @@ -90,4 +91,10 @@ std::unique_ptr SimpleSourceGroup::clone() const { return std::unique_ptr(std::move(cloned)); } +void SimpleSourceGroup::visitProperties(const PropertyVisitor& visitor) { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} + } // namespace SourceXtractor diff --git a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp index 0ebc3bb1f..21d3ef5c0 100644 --- a/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceGroupWithOnDemandProperties.cpp @@ -98,7 +98,7 @@ const Property& SourceGroupWithOnDemandProperties::getProperty(const PropertyId& throw PropertyNotFoundException(property_id); } -void SourceGroupWithOnDemandProperties::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SourceGroupWithOnDemandProperties::setProperty(std::shared_ptr property, const PropertyId& property_id) { m_property_holder.setProperty(std::move(property), property_id); } @@ -123,6 +123,20 @@ std::unique_ptr SourceGroupWithOnDemandProperties::clone() cons return std::unique_ptr(std::move(cloned)); } +void SourceGroupWithOnDemandProperties::EntangledSource::visitProperties(const PropertyVisitor& visitor) { + m_group.visitProperties(visitor); + m_source->visitProperties(visitor); + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} + +void SourceGroupWithOnDemandProperties::visitProperties(const PropertyVisitor& visitor) { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} + } // SourceXtractor namespace diff --git a/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp b/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp index e12ad8b19..507158a97 100644 --- a/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp +++ b/SEFramework/src/lib/Source/SourceWithOnDemandProperties.cpp @@ -54,7 +54,7 @@ const Property& SourceWithOnDemandProperties::getProperty(const PropertyId& prop throw PropertyNotFoundException(property_id); } -void SourceWithOnDemandProperties::setProperty(std::unique_ptr property, const PropertyId& property_id) { +void SourceWithOnDemandProperties::setProperty(std::shared_ptr property, const PropertyId& property_id) { // just forward to the ObjectWithProperties implementation m_property_holder.setProperty(std::move(property), property_id); } @@ -63,6 +63,11 @@ std::unique_ptr SourceWithOnDemandProperties::clone() const { return std::unique_ptr(new SourceWithOnDemandProperties(*this)); } +void SourceWithOnDemandProperties::visitProperties(const PropertyVisitor& visitor) { + std::for_each( + m_property_holder.begin(), m_property_holder.end(), + [visitor](const std::pair>& prop) { visitor(prop.first, prop.second); }); +} } // SEFramework namespace diff --git a/SEFramework/tests/src/Source/SourceInterface_test.cpp b/SEFramework/tests/src/Source/SourceInterface_test.cpp index 57dbe00df..311588741 100644 --- a/SEFramework/tests/src/Source/SourceInterface_test.cpp +++ b/SEFramework/tests/src/Source/SourceInterface_test.cpp @@ -52,7 +52,9 @@ class MockSourceInterface : public SourceInterface { using SourceInterface::setProperty; MOCK_CONST_METHOD1(getProperty, Property& (const PropertyId& property_id)); - void setProperty(std::unique_ptr, const PropertyId& ) {} + void setProperty(std::shared_ptr, const PropertyId& ) {} + + void visitProperties(const PropertyVisitor&) override {} std::unique_ptr clone() const override { return nullptr; From e8c75c88916eb416544dd4515742bdd4c9ce8746 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 5 Sep 2022 15:38:49 +0200 Subject: [PATCH 05/35] Add some consts to OutputConfig --- .../SEImplementation/Configuration/OutputConfig.h | 9 +++++---- .../src/lib/Configuration/OutputConfig.cpp | 13 +++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/SEImplementation/SEImplementation/Configuration/OutputConfig.h b/SEImplementation/SEImplementation/Configuration/OutputConfig.h index a6f48427d..95adf6b87 100644 --- a/SEImplementation/SEImplementation/Configuration/OutputConfig.h +++ b/SEImplementation/SEImplementation/Configuration/OutputConfig.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -53,11 +54,11 @@ class OutputConfig : public Euclid::Configuration::Configuration { void initialize(const UserValues& args) override; - std::string getOutputFile(); + std::string getOutputFile() const; - OutputFileFormat getOutputFileFormat(); + OutputFileFormat getOutputFileFormat() const; - const std::vector getOutputProperties(); + const std::vector getOutputProperties() const; size_t getFlushSize() const; diff --git a/SEImplementation/src/lib/Configuration/OutputConfig.cpp b/SEImplementation/src/lib/Configuration/OutputConfig.cpp index 1f9457ade..258f264ad 100644 --- a/SEImplementation/src/lib/Configuration/OutputConfig.cpp +++ b/SEImplementation/src/lib/Configuration/OutputConfig.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -76,13 +77,13 @@ void OutputConfig::preInitialize(const UserValues& args) { void OutputConfig::initialize(const UserValues& args) { m_out_file = args.at(OUTPUT_FILE).as(); - + std::stringstream properties_str {args.at(OUTPUT_PROPERTIES).as()}; std::string name; while (std::getline(properties_str, name, ',')) { m_output_properties.emplace_back(name); } - + auto format_name = boost::to_upper_copy(args.at(OUTPUT_FILE_FORMAT).as()); m_format = format_map.at(format_name); @@ -92,15 +93,15 @@ void OutputConfig::initialize(const UserValues& args) { m_unsorted = !args.at(OUTPUT_SORTED).as(); } -std::string OutputConfig::getOutputFile() { +std::string OutputConfig::getOutputFile() const { return m_out_file; } -OutputConfig::OutputFileFormat OutputConfig::getOutputFileFormat() { +OutputConfig::OutputFileFormat OutputConfig::getOutputFileFormat() const { return m_format; } -const std::vector OutputConfig::getOutputProperties() { +const std::vector OutputConfig::getOutputProperties() const { return m_output_properties; } From 51cb42eb4e0f1684fe2401c2070002a5cae47f36 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Wed, 17 Aug 2022 11:03:44 +0200 Subject: [PATCH 06/35] Expose column converters --- .../SEFramework/Output/OutputRegistry.h | 66 +++++++++++-------- SEFramework/SEFramework/Property/PropertyId.h | 8 +-- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/SEFramework/SEFramework/Output/OutputRegistry.h b/SEFramework/SEFramework/Output/OutputRegistry.h index 8fbaede43..8cb761a19 100644 --- a/SEFramework/SEFramework/Output/OutputRegistry.h +++ b/SEFramework/SEFramework/Output/OutputRegistry.h @@ -34,18 +34,40 @@ namespace SourceXtractor { class OutputRegistry { - public: - template using ColumnConverter = std::function; using SourceToRowConverter = std::function; + class ColumnFromSource { + public: + template + explicit ColumnFromSource(ColumnConverter converter) { + m_convert_func = [converter](const SourceInterface& source, std::size_t i) { + return converter(source.getProperty(i)); + }; + } + Euclid::Table::Row::cell_type operator()(const SourceInterface& source) const { + return m_convert_func(source, index); + } + std::size_t index = 0; + + private: + std::function m_convert_func; + }; + + struct ColInfo { + std::string unit; + std::string description; + }; + +public: template void registerColumnConverter(std::string column_name, ColumnConverter converter, std::string column_unit="", std::string column_description="") { m_property_to_names_map[typeid(PropertyType)].emplace_back(column_name); + m_name_to_property_map.emplace(column_name, typeid(PropertyType)); std::type_index conv_out_type = typeid(OutType); ColumnFromSource conv_func {converter}; m_name_to_converter_map.emplace(column_name, @@ -53,6 +75,14 @@ class OutputRegistry { m_name_to_col_info_map.emplace(column_name, ColInfo{column_unit, column_description}); } + std::type_index getPropertyForColumn(const std::string& column_name) const { + return m_name_to_property_map.at(column_name); + } + + const std::pair& getColumnConverter(const std::string& column_name) const { + return m_name_to_converter_map.at(column_name); + } + /** * When there are multiple instances of a given property, generate one column output with the given suffix for each * instance @@ -159,33 +189,11 @@ class OutputRegistry { void printPropertyColumnMap(const std::vector& properties={}); private: - - class ColumnFromSource { - public: - template - explicit ColumnFromSource(ColumnConverter converter) { - m_convert_func = [converter](const SourceInterface& source, std::size_t i){ - return converter(source.getProperty(i)); - }; - } - Euclid::Table::Row::cell_type operator()(const SourceInterface& source) { - return m_convert_func(source, index); - } - std::size_t index = 0; - private: - std::function m_convert_func; - }; - - struct ColInfo { - std::string unit; - std::string description; - }; - - std::map> m_property_to_names_map {}; - std::map> m_name_to_converter_map {}; - std::map m_name_to_col_info_map {}; - std::multimap m_output_properties {}; - + std::map> m_property_to_names_map{}; + std::map m_name_to_property_map{}; + std::map> m_name_to_converter_map{}; + std::map m_name_to_col_info_map{}; + std::multimap m_output_properties{}; }; } /* namespace SourceXtractor */ diff --git a/SEFramework/SEFramework/Property/PropertyId.h b/SEFramework/SEFramework/Property/PropertyId.h index dae899553..aa1b96e52 100644 --- a/SEFramework/SEFramework/Property/PropertyId.h +++ b/SEFramework/SEFramework/Property/PropertyId.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -39,6 +40,8 @@ namespace SourceXtractor { class PropertyId { public: + PropertyId(std::type_index type_id, unsigned int index) : m_type_id(type_id), m_index(index) {} + /// Templated factory method used to create a PropertyId based on the type of a property. /// An optional index parameter is used to make the distinction between several properties of the same type. template @@ -74,12 +77,9 @@ class PropertyId { std::string getString() const; private: - PropertyId(std::type_index type_id, unsigned int index) : m_type_id(type_id), m_index(index) {} std::type_index m_type_id; unsigned int m_index; - - friend struct std::hash; }; From 44206041b6df51d7268631b5a82a93e2cec7c2de Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 9 Sep 2022 11:21:04 +0200 Subject: [PATCH 07/35] Expose detection stamp --- .../DetectionFrameSourceStampPlugin.cpp | 44 +++++++++++++++---- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp b/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp index 004888d30..44f8720bf 100644 --- a/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -21,26 +22,51 @@ * Author: mschefer */ +#include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.h" +#include "NdArray/Borrowed.h" #include "SEFramework/Plugin/StaticPlugin.h" - #include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStamp.h" #include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampTaskFactory.h" -#include "SEImplementation/Plugin/DetectionFrameSourceStamp/DetectionFrameSourceStampPlugin.h" - namespace SourceXtractor { +using Euclid::NdArray::BorrowedRange; +using Euclid::NdArray::NdArray; + static StaticPlugin detection_frame_source_stamp_plugin; +struct StampConverter { + using DetectionVectorImage = DetectionFrameSourceStamp::DetectionVectorImage; + using PixelType = DetectionVectorImage::PixelType; + + std::function m_getter; + + NdArray operator()(const DetectionFrameSourceStamp& property) const { + auto& stamp = m_getter(property); + auto& data = const_cast&>(stamp.getData()); + std::vector shape{static_cast(stamp.getHeight()), static_cast(stamp.getWidth())}; + return NdArray(shape, BorrowedRange{data.data(), data.size()}); + } +}; + void DetectionFrameSourceStampPlugin::registerPlugin(PluginAPI& plugin_api) { - plugin_api.getTaskFactoryRegistry().registerTaskFactory(); + using PixelType = DetectionImage::PixelType; + + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_stamp", StampConverter{&DetectionFrameSourceStamp::getStamp}); + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_filtered_stamp", StampConverter{&DetectionFrameSourceStamp::getFilteredStamp}); + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_variance_stamp", StampConverter{&DetectionFrameSourceStamp::getVarianceStamp}); + plugin_api.getOutputRegistry().registerColumnConverter>( + "detection_thresholded_stamp", StampConverter{&DetectionFrameSourceStamp::getThresholdedStamp}); + + plugin_api.getTaskFactoryRegistry() + .registerTaskFactory(); } std::string DetectionFrameSourceStampPlugin::getIdString() const { return "DetectionFrameSourceStamp"; } -} - - - +} // namespace SourceXtractor From 355445f238a8de7b4ee5426d443fae0aff9a9d6b Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 16 Sep 2022 09:59:32 +0200 Subject: [PATCH 08/35] Allow to get property columns by ID and name --- .../SEFramework/Output/OutputRegistry.h | 7 +++- SEFramework/src/lib/Output/OutputRegistry.cpp | 40 ++++++++++++++----- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/SEFramework/SEFramework/Output/OutputRegistry.h b/SEFramework/SEFramework/Output/OutputRegistry.h index 8cb761a19..ce0486a7f 100644 --- a/SEFramework/SEFramework/Output/OutputRegistry.h +++ b/SEFramework/SEFramework/Output/OutputRegistry.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -184,6 +185,10 @@ class OutputRegistry { return result; } + std::set getColumns(const std::string& property) const; + + std::set getColumns(const PropertyId& property) const; + SourceToRowConverter getSourceToRowConverter(const std::vector& enabled_optional); void printPropertyColumnMap(const std::vector& properties={}); diff --git a/SEFramework/src/lib/Output/OutputRegistry.cpp b/SEFramework/src/lib/Output/OutputRegistry.cpp index 6cef389b1..31132b725 100644 --- a/SEFramework/src/lib/Output/OutputRegistry.cpp +++ b/SEFramework/src/lib/Output/OutputRegistry.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -33,8 +34,9 @@ using namespace Euclid::Table; namespace SourceXtractor { -auto OutputRegistry::getSourceToRowConverter(const std::vector& enabled_properties) -> SourceToRowConverter { - std::vector out_prop_list {}; +auto OutputRegistry::getSourceToRowConverter(const std::vector& enabled_properties) + -> SourceToRowConverter { + std::vector out_prop_list{}; for (auto& prop : enabled_properties) { if (m_output_properties.count(prop) == 0) { throw Elements::Exception() << "Unknown output property " << prop; @@ -47,28 +49,27 @@ auto OutputRegistry::getSourceToRowConverter(const std::vector& ena } } return [this, out_prop_list](const SourceInterface& source) { - std::vector info_list {}; - std::vector cell_values {}; + std::vector info_list{}; + std::vector cell_values{}; for (const auto& property : out_prop_list) { if (m_property_to_names_map.count(property) == 0) { throw Elements::Exception() << "Missing column generator for " << property.name(); } for (const auto& name : m_property_to_names_map.at(property)) { auto& col_info = m_name_to_col_info_map.at(name); - info_list.emplace_back(name, m_name_to_converter_map.at(name).first, - col_info.unit, col_info.description); + info_list.emplace_back(name, m_name_to_converter_map.at(name).first, col_info.unit, col_info.description); cell_values.emplace_back(m_name_to_converter_map.at(name).second(source)); } } if (info_list.empty()) { throw Elements::Exception() << "The given configuration would not generate any output"; } - return Row {std::move(cell_values), std::make_shared(move(info_list))}; + return Row{std::move(cell_values), std::make_shared(move(info_list))}; }; } void OutputRegistry::printPropertyColumnMap(const std::vector& properties) { - std::set properties_set {properties.begin(), properties.end()}; + std::set properties_set{properties.begin(), properties.end()}; for (auto& prop : m_output_properties) { if (!properties.empty() && properties_set.find(prop.first) == properties_set.end()) { continue; @@ -81,12 +82,31 @@ void OutputRegistry::printPropertyColumnMap(const std::vector& prop std::cout << " : " << info.description; } if (info.unit != "") { - std::cout << " " << info.unit << ""; // place here braces "()" around the units, if desired + std::cout << " " << info.unit << ""; // place here braces "()" around the units, if desired } std::cout << '\n'; } } } +std::set OutputRegistry::getColumns(const std::string& property) const { + std::set columns; + auto iter_range = m_output_properties.equal_range(property); + for (auto i = iter_range.first; i != iter_range.second; ++i) { + const auto& attrs = m_property_to_names_map.at(i->second); + std::copy(attrs.begin(), attrs.end(), std::inserter(columns, columns.begin())); + } + + return columns; +} + +std::set OutputRegistry::getColumns(const PropertyId& property) const { + auto iter = m_property_to_names_map.find(property.getTypeId()); + if (iter != m_property_to_names_map.end()) { + return {iter->second.begin(), iter->second.end()}; + } + return {}; } + +} // namespace SourceXtractor From 28cf91a4f4d6d7a475836d4afa411168d3006db0 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 16 Sep 2022 10:50:54 +0200 Subject: [PATCH 09/35] Expose converters for internal properties --- SEImplementation/CMakeLists.txt | 1 + .../Property/RegisterConverters.h | 30 ++++++++++++++ .../DetectionFrameInfoPlugin.cpp | 14 ++++--- .../src/lib/Property/RegisterConverters.cpp | 40 +++++++++++++++++++ 4 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 SEImplementation/SEImplementation/Property/RegisterConverters.h create mode 100644 SEImplementation/src/lib/Property/RegisterConverters.cpp diff --git a/SEImplementation/CMakeLists.txt b/SEImplementation/CMakeLists.txt index 1b52a93d3..3210fcd1e 100644 --- a/SEImplementation/CMakeLists.txt +++ b/SEImplementation/CMakeLists.txt @@ -96,6 +96,7 @@ elements_add_library(SEImplementation src/lib/Image/*.cpp ${COMMON_SRC} src/lib/Prefetcher/*.cpp + src/lib/Property/*.cpp ${PLUGIN_SRC} ${SE_PYTHON_SRC} LINK_LIBRARIES diff --git a/SEImplementation/SEImplementation/Property/RegisterConverters.h b/SEImplementation/SEImplementation/Property/RegisterConverters.h new file mode 100644 index 000000000..761650e3f --- /dev/null +++ b/SEImplementation/SEImplementation/Property/RegisterConverters.h @@ -0,0 +1,30 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef _SEIMPLEMENTATION_PROPERTY_CONVERTERS_H +#define _SEIMPLEMENTATION_PROPERTY_CONVERTERS_H + +#include "SEFramework/Output/OutputRegistry.h" + +namespace SourceXtractor { + +void registerDefaultConverters(OutputRegistry& registry); + +} + +#endif // _SEIMPLEMENTATION_PROPERTY_CONVERTERS_H diff --git a/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp b/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp index d806cc303..4e2b183d2 100644 --- a/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -17,10 +18,10 @@ #include "SEFramework/Plugin/StaticPlugin.h" +#include "SEImplementation/Image/ImageInterfaceTraits.h" #include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfo.h" -#include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfoTaskFactory.h" #include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfoPlugin.h" -#include "SEImplementation/Image/ImageInterfaceTraits.h" +#include "SEImplementation/Plugin/DetectionFrameInfo/DetectionFrameInfoTaskFactory.h" namespace SourceXtractor { @@ -28,12 +29,13 @@ static StaticPlugin detection_frame_info_plugin; void DetectionFrameInfoPlugin::registerPlugin(PluginAPI& plugin_api) { plugin_api.getTaskFactoryRegistry().registerTaskFactory(); + + plugin_api.getOutputRegistry().registerColumnConverter( + "detection_frame_hdu", [](const DetectionFrameInfo& frame_info) { return frame_info.getHduIndex(); }); } std::string DetectionFrameInfoPlugin::getIdString() const { return "DetectionFrameInfo"; } -} - - +} // namespace SourceXtractor diff --git a/SEImplementation/src/lib/Property/RegisterConverters.cpp b/SEImplementation/src/lib/Property/RegisterConverters.cpp new file mode 100644 index 000000000..131f405c5 --- /dev/null +++ b/SEImplementation/src/lib/Property/RegisterConverters.cpp @@ -0,0 +1,40 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEImplementation/Property/RegisterConverters.h" +#include "SEImplementation/Property/PixelCoordinateList.h" +#include "SEImplementation/Property/SourceId.h" + +using Euclid::NdArray::NdArray; + +namespace SourceXtractor { + +void registerDefaultConverters(OutputRegistry& registry) { + registry.registerColumnConverter>( + "pixel_coordinates", [](const PixelCoordinateList& coordinates) { + const auto& list = coordinates.getCoordinateList(); + NdArray coords({list.size(), 2}); + for (size_t i = 0; i < list.size(); ++i) { + coords.at(i, 0) = list[i].m_x; + coords.at(i, 1) = list[i].m_y; + } + return coords; + }); +} + +} // namespace SourceXtractor From f16826551f869aad35150cc39ce8c07b00ab5f83 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Wed, 21 Sep 2022 14:58:16 +0200 Subject: [PATCH 10/35] Grouping should forward events --- SEFramework/src/lib/Pipeline/SourceGrouping.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SEFramework/src/lib/Pipeline/SourceGrouping.cpp b/SEFramework/src/lib/Pipeline/SourceGrouping.cpp index 695b1035d..784acc673 100644 --- a/SEFramework/src/lib/Pipeline/SourceGrouping.cpp +++ b/SEFramework/src/lib/Pipeline/SourceGrouping.cpp @@ -103,6 +103,8 @@ void SourceGrouping::receiveProcessSignal(const ProcessSourcesEvent& process_eve sendSource(std::move(*group)); m_source_groups.erase(group); } + + sendProcessSignal(process_event); } std::set SourceGrouping::requiredProperties() const { From 48f11474aa1ab9bcc626f9eed7ad340667d89932 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Tue, 20 Sep 2022 10:56:53 +0200 Subject: [PATCH 11/35] Rename SourceXtractorPy => SEPythonConfig Making space for a module callable from Python --- SEImplementation/CMakeLists.txt | 2 +- SEImplementation/python/sourcextractor/__init__.py | 2 +- .../python/sourcextractor/config/compat.py | 2 +- .../python/sourcextractor/config/measurement_config.py | 10 +++++----- .../python/sourcextractor/config/measurement_images.py | 2 +- .../python/sourcextractor/config/model_fitting.py | 2 +- .../src/lib/PythonConfig/PythonInterpreter.cpp | 4 ++-- SEImplementation/src/lib/PythonConfig/PythonModule.cpp | 7 ++++--- doc/src/conf.py | 4 ++-- 9 files changed, 18 insertions(+), 17 deletions(-) diff --git a/SEImplementation/CMakeLists.txt b/SEImplementation/CMakeLists.txt index 3210fcd1e..e3bbb4e90 100644 --- a/SEImplementation/CMakeLists.txt +++ b/SEImplementation/CMakeLists.txt @@ -111,7 +111,7 @@ elements_add_library(SEImplementation #=============================================================================== # SEImplementation has a part that needs to be reachable from Python. #=============================================================================== -elements_add_python_module(SourceXtractorPy +elements_add_python_module(SEPythonConfig src/lib/PythonConfig/PythonModule.cpp LINK_LIBRARIES SEImplementation INCLUDE_DIRS ${Boost_INCLUDE_DIR} ${PYTHON_INCLUDE_DIRS} diff --git a/SEImplementation/python/sourcextractor/__init__.py b/SEImplementation/python/sourcextractor/__init__.py index 978074d20..16a48b8d4 100644 --- a/SEImplementation/python/sourcextractor/__init__.py +++ b/SEImplementation/python/sourcextractor/__init__.py @@ -15,5 +15,5 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from _SourceXtractorPy import Flags +from _SEPythonConfig import Flags diff --git a/SEImplementation/python/sourcextractor/config/compat.py b/SEImplementation/python/sourcextractor/config/compat.py index 24320891f..1801f9bfd 100644 --- a/SEImplementation/python/sourcextractor/config/compat.py +++ b/SEImplementation/python/sourcextractor/config/compat.py @@ -17,7 +17,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import sys -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp from .measurement_config import MeasurementConfig, global_measurement_config from .model_fitting import ModelFitting diff --git a/SEImplementation/python/sourcextractor/config/measurement_config.py b/SEImplementation/python/sourcextractor/config/measurement_config.py index c2756340d..a208b751d 100644 --- a/SEImplementation/python/sourcextractor/config/measurement_config.py +++ b/SEImplementation/python/sourcextractor/config/measurement_config.py @@ -19,7 +19,7 @@ import sys -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp from .measurement_images import DataCubeSlice, FitsFile, ImageGroup, MeasurementGroup, \ MeasurementImage @@ -88,7 +88,7 @@ def load_fits_image(self, image, psf=None, weight=None, **kwargs): image_file = FitsFile(image) if "image_hdu" in kwargs.keys(): image_hdu_list = [kwargs.pop("image_hdu")] - else: + else: image_hdu_list = image_file.hdu_list # handles the PSFs @@ -99,8 +99,8 @@ def load_fits_image(self, image, psf=None, weight=None, **kwargs): psf_hdu_list = [0] * len(psf_file_list) else: if "psf_hdu" in kwargs.keys(): - psf_hdu_list = [kwargs.pop("psf_hdu")] * len(image_hdu_list) - else: + psf_hdu_list = [kwargs.pop("psf_hdu")] * len(image_hdu_list) + else: psf_hdu_list = range(len(image_hdu_list)) psf_file_list = [psf] * len(image_hdu_list) @@ -117,7 +117,7 @@ def load_fits_image(self, image, psf=None, weight=None, **kwargs): weight_file = FitsFile(weight) if "weight_hdu" in kwargs.keys(): weight_hdu_list = [kwargs.pop("weight_hdu")] * len(image_hdu_list) - else: + else: weight_hdu_list = weight_file.hdu_list weight_file_list = [weight_file] * len(image_hdu_list) diff --git a/SEImplementation/python/sourcextractor/config/measurement_images.py b/SEImplementation/python/sourcextractor/config/measurement_images.py index fb7c49fc7..5e1ba778e 100644 --- a/SEImplementation/python/sourcextractor/config/measurement_images.py +++ b/SEImplementation/python/sourcextractor/config/measurement_images.py @@ -21,7 +21,7 @@ import re import sys -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp if sys.version_info.major < 3: from StringIO import StringIO diff --git a/SEImplementation/python/sourcextractor/config/model_fitting.py b/SEImplementation/python/sourcextractor/config/model_fitting.py index b27d76e56..b652b1660 100644 --- a/SEImplementation/python/sourcextractor/config/model_fitting.py +++ b/SEImplementation/python/sourcextractor/config/model_fitting.py @@ -22,7 +22,7 @@ import warnings from enum import Enum -import _SourceXtractorPy as cpp +import _SEPythonConfig as cpp try: import pyston diff --git a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp index ac0b3ae89..797e0169c 100644 --- a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp +++ b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -94,7 +94,7 @@ void PythonInterpreter::runFile(const std::string& filename, const std::vector */ @@ -36,7 +37,7 @@ namespace bp = boost::python; namespace SourceXtractor { -BOOST_PYTHON_MODULE(_SourceXtractorPy) { +BOOST_PYTHON_MODULE(_SEPythonConfig) { bp::class_("OutputWrapper", "A file-like object used to wrap stdout and stderr", bp::no_init) diff --git a/doc/src/conf.py b/doc/src/conf.py index 70c8f2597..74570a709 100644 --- a/doc/src/conf.py +++ b/doc/src/conf.py @@ -134,7 +134,7 @@ # unit titles (such as .. function::). add_module_names = False -autodoc_mock_imports = ['.measurement_images', '_SourceXtractorPy'] +autodoc_mock_imports = ['.measurement_images', '_SEPythonConfig'] # -- Options for HTML output ---------------------------------------------- @@ -466,7 +466,7 @@ def setup(app): # -- Options for pybtex ---------------------------------------------- bibtex_bibfiles = ["references.bib"] - + from packaging import version as vers from pybtex import __version__ as pybtex_version from pybtex.style.formatting.unsrt import Style as UnsrtStyle, date, pages, toplevel From 55ca5a798acb620056b8f38e621cb984bfc8ce4b Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Tue, 20 Sep 2022 11:57:05 +0200 Subject: [PATCH 12/35] Move pure Python code to a separate module --- SEImplementation/CMakeLists.txt | 1 - SEPythonWrapper/CMakeLists.txt | 73 +++++++++++++++++++ .../python/sourcextractor/__init__.py | 0 .../python/sourcextractor/config/__init__.py | 0 .../python/sourcextractor/config/argv.py | 0 .../python/sourcextractor/config/compat.py | 0 .../config/measurement_config.py | 0 .../config/measurement_images.py | 0 .../sourcextractor/config/model_fitting.py | 0 9 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 SEPythonWrapper/CMakeLists.txt rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/__init__.py (100%) rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/config/__init__.py (100%) rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/config/argv.py (100%) rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/config/compat.py (100%) rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/config/measurement_config.py (100%) rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/config/measurement_images.py (100%) rename {SEImplementation => SEPythonWrapper}/python/sourcextractor/config/model_fitting.py (100%) diff --git a/SEImplementation/CMakeLists.txt b/SEImplementation/CMakeLists.txt index e3bbb4e90..1ef3bc573 100644 --- a/SEImplementation/CMakeLists.txt +++ b/SEImplementation/CMakeLists.txt @@ -223,7 +223,6 @@ elements_add_unit_test(AssocMode_test tests/src/Plugin/AssocMode/AssocMode_test. # elements_install_python_modules() # elements_install_scripts() #=============================================================================== -elements_install_python_modules() #=============================================================================== # Add the elements_install_conf_files macro diff --git a/SEPythonWrapper/CMakeLists.txt b/SEPythonWrapper/CMakeLists.txt new file mode 100644 index 000000000..bcef6129e --- /dev/null +++ b/SEPythonWrapper/CMakeLists.txt @@ -0,0 +1,73 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.5) + +#=============================================================================== +# Load elements_subdir macro here +# Examples: +# For declaring a project module: +# elements_subdir(ElementsExamples) +#=============================================================================== +elements_subdir(SEPythonWrapper) + +#=============================================================================== +# Load elements_depends_on_subdirs macro here +# For creating a dependency onto an other accessible module +# elements_depends_on_subdirs(ElementsKernel) +#=============================================================================== +elements_depends_on_subdirs(SEImplementation) + +#=============================================================================== +# Add the find_package macro (a pure CMake command) here to locate the +# libraries. +# Examples: +# find_package(CppUnit) +#=============================================================================== + +#=============================================================================== +# Declare the library dependencies here +# Example: +# elements_add_library(ElementsExamples src/Lib/*.cpp +# LINK_LIBRARIES Boost ElementsKernel +# INCLUDE_DIRS Boost ElementsKernel +# PUBLIC_HEADERS ElementsExamples) +#=============================================================================== + +#=============================================================================== +# SEImplementation has a part that needs to be reachable from Python. +#=============================================================================== + +#=============================================================================== +# Declare the executables here +# Example: +# elements_add_executable(ElementsProgramExample src/Program/ProgramExample.cpp +# LINK_LIBRARIES Boost ElementsExamples +# INCLUDE_DIRS Boost ElementsExamples) +#=============================================================================== + +#=============================================================================== +# Declare the Boost tests here +# Example: +# elements_add_unit_test(BoostClassExample tests/src/Boost/ClassExample_test.cpp +# EXECUTABLE BoostClassExample_test +# INCLUDE_DIRS ElementsExamples +# LINK_LIBRARIES ElementsExamples TYPE Boost) +#=============================================================================== + +#=============================================================================== +# Declare the Python programs here +# Examples : +# elements_add_python_program(PythonProgramExample +# ElementsExamples.PythonProgramExample) +#=============================================================================== + +#=============================================================================== +# Use the following macro for python modules, scripts and aux files: +# elements_install_python_modules() +# elements_install_scripts() +#=============================================================================== +elements_install_python_modules() + +#=============================================================================== +# Add the elements_install_conf_files macro +# Examples: +# elements_install_conf_files() +#=============================================================================== diff --git a/SEImplementation/python/sourcextractor/__init__.py b/SEPythonWrapper/python/sourcextractor/__init__.py similarity index 100% rename from SEImplementation/python/sourcextractor/__init__.py rename to SEPythonWrapper/python/sourcextractor/__init__.py diff --git a/SEImplementation/python/sourcextractor/config/__init__.py b/SEPythonWrapper/python/sourcextractor/config/__init__.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/__init__.py rename to SEPythonWrapper/python/sourcextractor/config/__init__.py diff --git a/SEImplementation/python/sourcextractor/config/argv.py b/SEPythonWrapper/python/sourcextractor/config/argv.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/argv.py rename to SEPythonWrapper/python/sourcextractor/config/argv.py diff --git a/SEImplementation/python/sourcextractor/config/compat.py b/SEPythonWrapper/python/sourcextractor/config/compat.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/compat.py rename to SEPythonWrapper/python/sourcextractor/config/compat.py diff --git a/SEImplementation/python/sourcextractor/config/measurement_config.py b/SEPythonWrapper/python/sourcextractor/config/measurement_config.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/measurement_config.py rename to SEPythonWrapper/python/sourcextractor/config/measurement_config.py diff --git a/SEImplementation/python/sourcextractor/config/measurement_images.py b/SEPythonWrapper/python/sourcextractor/config/measurement_images.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/measurement_images.py rename to SEPythonWrapper/python/sourcextractor/config/measurement_images.py diff --git a/SEImplementation/python/sourcextractor/config/model_fitting.py b/SEPythonWrapper/python/sourcextractor/config/model_fitting.py similarity index 100% rename from SEImplementation/python/sourcextractor/config/model_fitting.py rename to SEPythonWrapper/python/sourcextractor/config/model_fitting.py From 1d533fb3dab4bd980cdcdc4f9f603804bac281d3 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 15 Aug 2022 16:31:46 +0200 Subject: [PATCH 13/35] Module to call sourcex from Python --- .../SEFramework/Output/OutputRegistry.h | 2 +- .../Configuration/PythonConfig.h | 5 +- .../PythonConfig/PythonInterpreter.h | 2 +- .../src/lib/Configuration/PythonConfig.cpp | 56 ++--- .../DetectionFramePixelValuesPlugin.cpp | 16 +- .../FlexibleModelFittingTaskFactory.cpp | 4 +- .../lib/PythonConfig/PythonInterpreter.cpp | 12 +- SEMain/src/program/SourceXtractor.cpp | 2 - SEPythonModule/CMakeLists.txt | 79 +++++++ SEPythonModule/SEPythonModule/ConfigAdapter.h | 51 +++++ SEPythonModule/SEPythonModule/Context.h | 104 ++++++++++ SEPythonModule/SEPythonModule/Deblending.h | 50 +++++ SEPythonModule/SEPythonModule/FitsOutput.h | 56 +++++ SEPythonModule/SEPythonModule/Grouping.h | 49 +++++ SEPythonModule/SEPythonModule/Measurement.h | 51 +++++ SEPythonModule/SEPythonModule/NumpyOutput.h | 54 +++++ SEPythonModule/SEPythonModule/Partition.h | 50 +++++ .../SEPythonModule/PipelineReceiver.h | 91 +++++++++ SEPythonModule/SEPythonModule/Segmentation.h | 52 +++++ .../SEPythonModule/SourceInterface.h | 83 ++++++++ SEPythonModule/src/lib/ConfigAdapter.cpp | 62 ++++++ SEPythonModule/src/lib/Context.cpp | 140 +++++++++++++ SEPythonModule/src/lib/Deblending.cpp | 70 +++++++ SEPythonModule/src/lib/FitsOutput.cpp | 77 +++++++ SEPythonModule/src/lib/Grouping.cpp | 72 +++++++ SEPythonModule/src/lib/Measurement.cpp | 79 +++++++ SEPythonModule/src/lib/NumpyOutput.cpp | 79 +++++++ SEPythonModule/src/lib/Partition.cpp | 72 +++++++ SEPythonModule/src/lib/SePyMain.cpp | 146 +++++++++++++ SEPythonModule/src/lib/Segmentation.cpp | 78 +++++++ SEPythonModule/src/lib/SourceInterface.cpp | 192 ++++++++++++++++++ 31 files changed, 1896 insertions(+), 40 deletions(-) create mode 100644 SEPythonModule/CMakeLists.txt create mode 100644 SEPythonModule/SEPythonModule/ConfigAdapter.h create mode 100644 SEPythonModule/SEPythonModule/Context.h create mode 100644 SEPythonModule/SEPythonModule/Deblending.h create mode 100644 SEPythonModule/SEPythonModule/FitsOutput.h create mode 100644 SEPythonModule/SEPythonModule/Grouping.h create mode 100644 SEPythonModule/SEPythonModule/Measurement.h create mode 100644 SEPythonModule/SEPythonModule/NumpyOutput.h create mode 100644 SEPythonModule/SEPythonModule/Partition.h create mode 100644 SEPythonModule/SEPythonModule/PipelineReceiver.h create mode 100644 SEPythonModule/SEPythonModule/Segmentation.h create mode 100644 SEPythonModule/SEPythonModule/SourceInterface.h create mode 100644 SEPythonModule/src/lib/ConfigAdapter.cpp create mode 100644 SEPythonModule/src/lib/Context.cpp create mode 100644 SEPythonModule/src/lib/Deblending.cpp create mode 100644 SEPythonModule/src/lib/FitsOutput.cpp create mode 100644 SEPythonModule/src/lib/Grouping.cpp create mode 100644 SEPythonModule/src/lib/Measurement.cpp create mode 100644 SEPythonModule/src/lib/NumpyOutput.cpp create mode 100644 SEPythonModule/src/lib/Partition.cpp create mode 100644 SEPythonModule/src/lib/SePyMain.cpp create mode 100644 SEPythonModule/src/lib/Segmentation.cpp create mode 100644 SEPythonModule/src/lib/SourceInterface.cpp diff --git a/SEFramework/SEFramework/Output/OutputRegistry.h b/SEFramework/SEFramework/Output/OutputRegistry.h index ce0486a7f..c29aed901 100644 --- a/SEFramework/SEFramework/Output/OutputRegistry.h +++ b/SEFramework/SEFramework/Output/OutputRegistry.h @@ -177,7 +177,7 @@ class OutputRegistry { m_output_properties.emplace(alias_name, typeid(PropertyType)); } - std::set getOutputPropertyNames() { + std::set getOutputPropertyNames() const { std::set result {}; for (auto& pair : m_output_properties) { result.emplace(pair.first); diff --git a/SEImplementation/SEImplementation/Configuration/PythonConfig.h b/SEImplementation/SEImplementation/Configuration/PythonConfig.h index e738254c3..0177bd474 100644 --- a/SEImplementation/SEImplementation/Configuration/PythonConfig.h +++ b/SEImplementation/SEImplementation/Configuration/PythonConfig.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -41,6 +42,8 @@ class PythonConfig : public Euclid::Configuration::Configuration { PythonInterpreter& getInterpreter() const; +private: + boost::python::object m_measurement_config; }; } diff --git a/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h b/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h index a10e14368..3c4a4ed92 100644 --- a/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h +++ b/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h @@ -41,7 +41,7 @@ class PythonInterpreter { void runFile(const std::string& filename, const std::vector& argv); - void setupContext(); + void setupContext(boost::python::object config = {}); virtual ~PythonInterpreter(); diff --git a/SEImplementation/src/lib/Configuration/PythonConfig.cpp b/SEImplementation/src/lib/Configuration/PythonConfig.cpp index 347796a85..1129420d0 100644 --- a/SEImplementation/src/lib/Configuration/PythonConfig.cpp +++ b/SEImplementation/src/lib/Configuration/PythonConfig.cpp @@ -15,13 +15,13 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PythonConfig.cpp * @author Nikolaos Apostolakos */ -#include #include +#include using namespace Euclid::Configuration; namespace po = boost::program_options; @@ -29,10 +29,11 @@ namespace fs = boost::filesystem; namespace { -const std::string PYTHON_CONFIG_FILE { "python-config-file" }; -const std::string PYTHON_ARGV { "python-arg" }; +const std::string PYTHON_CONFIG_FILE{"python-config-file"}; +const std::string PYTHON_ARGV{"python-arg"}; +const std::string PYTHON_CONFIG_OBJ{"python-config-object"}; -} +} // namespace namespace SourceXtractor { @@ -41,38 +42,43 @@ PythonConfig::PythonConfig(long manager_id) : Configuration(manager_id) { } std::map PythonConfig::getProgramOptions() { - return {{"Measurement config", { - {PYTHON_CONFIG_FILE.c_str(), po::value()->default_value({}, ""), - "Measurements python configuration file"}, - {PYTHON_ARGV.c_str(), po::value>()->multitoken(), - "Parameters to pass to Python via sys.argv"} - }}}; + return {{"Measurement config", + {{PYTHON_CONFIG_FILE.c_str(), po::value()->default_value({}, ""), + "Measurements python configuration file"}, + {PYTHON_ARGV.c_str(), po::value>()->multitoken(), + "Parameters to pass to Python via sys.argv"}}}}; } - void PythonConfig::preInitialize(const UserValues& args) { - auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); - if (!filename.empty() && !fs::exists(filename)) { - throw Elements::Exception() << "Python configuration file " << filename - << " does not exist"; + auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); + auto py_obj_iter = args.find(PYTHON_CONFIG_OBJ); + + if (py_obj_iter != args.end()) { + m_measurement_config = py_obj_iter->second.as(); + } else if (!filename.empty() && !fs::exists(filename)) { + throw Elements::Exception() << "Python configuration file " << filename << " does not exist"; } } void PythonConfig::initialize(const UserValues& args) { - auto &singleton = PythonInterpreter::getSingleton(); - auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); - if (!filename.empty()) { - std::vector argv; - if (args.find(PYTHON_ARGV) != args.end()) { - argv = args.find(PYTHON_ARGV)->second.as>(); + auto& singleton = PythonInterpreter::getSingleton(); + if (m_measurement_config) { + singleton.setupContext(m_measurement_config); + } else { + auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); + if (!filename.empty()) { + std::vector argv; + if (args.find(PYTHON_ARGV) != args.end()) { + argv = args.find(PYTHON_ARGV)->second.as>(); + } + singleton.runFile(filename, argv); } - singleton.runFile(filename, argv); } - singleton.setupContext(); + singleton.setupContext(m_measurement_config); } PythonInterpreter& PythonConfig::getInterpreter() const { return PythonInterpreter::getSingleton(); } -} // end of namespace SourceXtractor +} // end of namespace SourceXtractor diff --git a/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp b/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp index a9a6614bc..ee37969e8 100644 --- a/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp +++ b/SEImplementation/src/lib/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -21,24 +22,25 @@ * Author: mschefer */ +#include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.h" #include "SEFramework/Plugin/StaticPlugin.h" - #include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValues.h" #include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesTaskFactory.h" -#include "SEImplementation/Plugin/DetectionFramePixelValues/DetectionFramePixelValuesPlugin.h" - namespace SourceXtractor { static StaticPlugin detection_frame_pixel_values_plugin; void DetectionFramePixelValuesPlugin::registerPlugin(PluginAPI& plugin_api) { - plugin_api.getTaskFactoryRegistry().registerTaskFactory(); + auto& output_registry = plugin_api.getOutputRegistry(); + output_registry.registerColumnConverter>( + "detection_pixel_values", [](const DetectionFramePixelValues& pixels) { return pixels.getValues(); }); + plugin_api.getTaskFactoryRegistry() + .registerTaskFactory(); } std::string DetectionFramePixelValuesPlugin::getIdString() const { return ""; } -} - +} // namespace SourceXtractor diff --git a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp index a5620ebd8..0e15c8ce3 100644 --- a/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp +++ b/SEImplementation/src/lib/Plugin/FlexibleModelFitting/FlexibleModelFittingTaskFactory.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/* + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -51,6 +52,7 @@ std::shared_ptr FlexibleModelFittingTaskFactory::createTask(const Property void FlexibleModelFittingTaskFactory::reportConfigDependencies(Euclid::Configuration::ConfigManager& manager) const { manager.registerConfiguration(); + manager.registerConfiguration(); } void FlexibleModelFittingTaskFactory::configure(Euclid::Configuration::ConfigManager& manager) { diff --git a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp index 797e0169c..c56a11fe6 100644 --- a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp +++ b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp @@ -50,6 +50,10 @@ PythonInterpreter& PythonInterpreter::getSingleton() { } PythonInterpreter::PythonInterpreter() : m_out_wrapper(stdout_logger), m_err_wrapper(stderr_logger) { + // We may be called *from* Python! + if (Py_IsInitialized()) { + return; + } // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt // Here we are not interested on this behaviour, so we get whatever handler we've got (normally // the default one) and restore it after initializing the interpreter @@ -121,10 +125,14 @@ void PythonInterpreter::runFile(const std::string& filename, const std::vector(); config_manager.registerConfiguration(); config_manager.registerConfiguration(); - config_manager.registerConfiguration(); config_manager.registerConfiguration(); CheckImages::getInstance().reportConfigDependencies(config_manager); diff --git a/SEPythonModule/CMakeLists.txt b/SEPythonModule/CMakeLists.txt new file mode 100644 index 000000000..61a225225 --- /dev/null +++ b/SEPythonModule/CMakeLists.txt @@ -0,0 +1,79 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8.12) + +#=============================================================================== +# Load elements_subdir macro here +# Examples: +# For declaring a project module: +# elements_subdir(ElementsExamples) +#=============================================================================== +elements_subdir(SEPythonModule) + +#=============================================================================== +# Load elements_depends_on_subdirs macro here +# For creating a dependency onto an other accessible module +# elements_depends_on_subdirs(ElementsKernel) +#=============================================================================== +elements_depends_on_subdirs(ElementsKernel) +elements_depends_on_subdirs(SEImplementation) +elements_depends_on_subdirs(SEMain) + +#=============================================================================== +# Add the find_package macro (a pure CMake command) here to locate the +# libraries. +# Examples: +# find_package(CppUnit) +#=============================================================================== +find_package(PythonInterp ${PYTHON_EXPLICIT_VERSION} COMPONENTS Development) +set(PYTHON_SUFFIX "${PYTHON_VERSION_MAJOR}${PYTHON_VERSION_MINOR}") +find_package(Boost 1.63 COMPONENTS "numpy${PYTHON_SUFFIX}" "python${PYTHON_SUFFIX}") + +if (Boost_FOUND) + #=============================================================================== + # Declare the library dependencies here + # Example: + # elements_add_library(ElementsExamples src/Lib/*.cpp + # INCLUDE_DIRS Boost ElementsKernel + # LINK_LIBRARIES Boost ElementsKernel + # PUBLIC_HEADERS ElementsExamples) + #=============================================================================== + elements_add_python_module(SEPythonModule + src/lib/*.cpp + LINK_LIBRARIES SEImplementation SEMain Boost PythonInterp + ) + + #=============================================================================== + # Declare the executables here + # Example: + # elements_add_executable(ElementsProgramExample src/Program/ProgramExample.cpp + # INCLUDE_DIRS Boost ElementsExamples + # LINK_LIBRARIES Boost ElementsExamples) + #=============================================================================== + + #=============================================================================== + # Declare the Boost tests here + # Example: + # elements_add_unit_test(BoostClassExample tests/src/Boost/ClassExample_test.cpp + # EXECUTABLE BoostClassExample_test + # INCLUDE_DIRS ElementsExamples + # LINK_LIBRARIES ElementsExamples TYPE Boost) + #=============================================================================== + + #=============================================================================== + # Use the following macro for python modules, scripts and aux files: + # elements_install_python_modules() + # elements_install_scripts() + #=============================================================================== + + #=============================================================================== + # Declare the Python programs here + # Examples : + # elements_add_python_program(PythonProgramExample + # ElementsExamples.PythonProgramExample) + #=============================================================================== + + #=============================================================================== + # Add the elements_install_conf_files macro + # Examples: + # elements_install_conf_files() + #=============================================================================== +endif (Boost_FOUND) diff --git a/SEPythonModule/SEPythonModule/ConfigAdapter.h b/SEPythonModule/SEPythonModule/ConfigAdapter.h new file mode 100644 index 000000000..cb789362f --- /dev/null +++ b/SEPythonModule/SEPythonModule/ConfigAdapter.h @@ -0,0 +1,51 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_CONFIGADAPTER_H +#define SOURCEXTRACTORPLUSPLUS_CONFIGADAPTER_H + +#include +#include +#include +#include + +namespace SourceXPy { + +class ConfigAdapter { +public: + using config_map_t = std::map; + + explicit ConfigAdapter(boost::program_options::options_description opt_desc); + + void fromPython(const boost::python::dict& config); + + template + void set(const std::string& key, T&& value) { + m_options[key] = boost::program_options::variable_value(std::forward(value), false); + } + + const config_map_t& getOptions() const; + +private: + boost::program_options::options_description m_option_descriptions; + config_map_t m_options; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_CONFIGADAPTER_H diff --git a/SEPythonModule/SEPythonModule/Context.h b/SEPythonModule/SEPythonModule/Context.h new file mode 100644 index 000000000..db2ce9fa6 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Context.h @@ -0,0 +1,104 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_CONTEXT_H +#define SOURCEXTRACTORPLUSPLUS_CONTEXT_H + +#include +#include +#include + +namespace Euclid { +namespace Configuration { +class ConfigManager; +} +} // namespace Euclid + +namespace SourceXtractor { +class TaskFactoryRegistry; +class TaskProvider; +class OutputRegistry; +class PluginManager; +class SegmentationFactory; +class SourceFactory; +class SourceGroupFactory; +class PartitionFactory; +class GroupingFactory; +class DeblendingFactory; +class OutputFactory; +class SourceWithOnDemandPropertiesFactory; +class MeasurementFactory; +class SourceInterface; +} // namespace SourceXtractor + +namespace SourceXPy { + +/** + * Wrap the required context for a sourcextractor++ run + */ +class Context { +public: + Context(const boost::python::dict& global_config, const boost::python::object& measurement_config); + Context(const Context&) = delete; + Context(Context&&) = default; + + boost::python::dict get_properties() const; + + std::shared_ptr m_config_manager; + std::shared_ptr m_task_factory_registry; + std::shared_ptr m_task_provider; + std::shared_ptr m_output_registry; + std::shared_ptr m_plugin_manager; + std::shared_ptr m_source_factory; + std::shared_ptr m_group_factory; + std::shared_ptr m_segmentation_factory; + std::shared_ptr m_partition_factory; + std::shared_ptr m_grouping_factory; + std::shared_ptr m_deblending_factory; + std::shared_ptr m_measurement_factory; + std::shared_ptr m_output_factory; + std::function m_source_to_row; + + static void enter(const std::shared_ptr&); + static void exit(const std::shared_ptr&, const boost::python::object& exc_type, + const boost::python::object& exc_value, const boost::python::object& traceback); + static const std::shared_ptr& get_global_context(); + +private: + static thread_local std::shared_ptr s_context; +}; + +class ContextPtr { +public: + ContextPtr(std::shared_ptr ptr) : m_ptr(std::move(ptr)) { // NOLINT: Allow implicit conversion + if (!m_ptr) { + m_ptr = Context::get_global_context(); + } + } + + Context* operator->() const { + return m_ptr.get(); + } + +private: + std::shared_ptr m_ptr; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_CONTEXT_H diff --git a/SEPythonModule/SEPythonModule/Deblending.h b/SEPythonModule/SEPythonModule/Deblending.h new file mode 100644 index 000000000..24538f9cc --- /dev/null +++ b/SEPythonModule/SEPythonModule/Deblending.h @@ -0,0 +1,50 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_DEBLENDING_H +#define SOURCEXTRACTORPLUSPLUS_DEBLENDING_H + +#include "SEFramework/Pipeline/Deblending.h" +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEPythonModule/Context.h" + +namespace SourceXPy { + +class Deblending : public SourceXtractor::PipelineReceiver { +public: + explicit Deblending(ContextPtr context); + ~Deblending() override = default; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr source) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj) const; + +private: + ContextPtr m_context; + std::shared_ptr m_deblending; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_DEBLENDING_H diff --git a/SEPythonModule/SEPythonModule/FitsOutput.h b/SEPythonModule/SEPythonModule/FitsOutput.h new file mode 100644 index 000000000..9e3655a32 --- /dev/null +++ b/SEPythonModule/SEPythonModule/FitsOutput.h @@ -0,0 +1,56 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_FITSOUTPUT_H +#define SOURCEXTRACTORPLUSPLUS_FITSOUTPUT_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEPythonModule/Context.h" +#include + +namespace SourceXtractor { +class Output; +} + +namespace SourceXPy { + +class FitsOutput : public SourceXtractor::PipelineReceiver { +public: + explicit FitsOutput(ContextPtr context); + ~FitsOutput() override; + + std::string repr() const; + + void receiveSource(std::unique_ptr group) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj); + + void get(); + +private: + ContextPtr m_context; + std::shared_ptr m_output; + Euclid::Semaphore m_semaphore{0}; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_FITSOUTPUT_H diff --git a/SEPythonModule/SEPythonModule/Grouping.h b/SEPythonModule/SEPythonModule/Grouping.h new file mode 100644 index 000000000..77c256044 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Grouping.h @@ -0,0 +1,49 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_GROUPING_H +#define SOURCEXTRACTORPLUSPLUS_GROUPING_H + +#include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEPythonModule/Context.h" + +namespace SourceXPy { + +class Grouping : public SourceXtractor::PipelineReceiver { +public: + explicit Grouping(ContextPtr context); + ~Grouping() override = default; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr source) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj) const; + +private: + ContextPtr m_context; + std::shared_ptr m_grouping; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_GROUPING_H diff --git a/SEPythonModule/SEPythonModule/Measurement.h b/SEPythonModule/SEPythonModule/Measurement.h new file mode 100644 index 000000000..55001ca99 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Measurement.h @@ -0,0 +1,51 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_MEASUREMENT_H +#define SOURCEXTRACTORPLUSPLUS_MEASUREMENT_H + +#include "SEFramework/Pipeline/Measurement.h" +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEPythonModule/Context.h" + +namespace SourceXPy { + +class Measurement : public SourceXtractor::PipelineReceiver { +public: + explicit Measurement(ContextPtr context); + ~Measurement() override; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr group) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj); + +private: + ContextPtr m_context; + std::unique_ptr m_measurement; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_MEASUREMENT_H diff --git a/SEPythonModule/SEPythonModule/NumpyOutput.h b/SEPythonModule/SEPythonModule/NumpyOutput.h new file mode 100644 index 000000000..b8e4e6754 --- /dev/null +++ b/SEPythonModule/SEPythonModule/NumpyOutput.h @@ -0,0 +1,54 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_NUMPYOUTPUT_H +#define SOURCEXTRACTORPLUSPLUS_NUMPYOUTPUT_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEPythonModule/Context.h" +#include +#include
+ +namespace SourceXPy { + +class NumpyOutput : public SourceXtractor::PipelineReceiver { +public: + explicit NumpyOutput(ContextPtr context); + ~NumpyOutput() override; + + std::string repr() const; + + void receiveSource(std::unique_ptr group) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj); + + boost::python::object getTable(); + +private: + ContextPtr m_context; + std::function m_source_to_row; + std::vector m_rows; + Euclid::Semaphore m_semaphore{0}; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_NUMPYOUTPUT_H diff --git a/SEPythonModule/SEPythonModule/Partition.h b/SEPythonModule/SEPythonModule/Partition.h new file mode 100644 index 000000000..d48b778f3 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Partition.h @@ -0,0 +1,50 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_PARTITION_H +#define SOURCEXTRACTORPLUSPLUS_PARTITION_H + +#include "SEFramework/Pipeline/Partition.h" +#include "SEPythonModule/Context.h" +#include + +namespace SourceXPy { + +class Partition : public SourceXtractor::PipelineReceiver { +public: + explicit Partition(ContextPtr context); + ~Partition() override = default; + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void receiveSource(std::unique_ptr source) override; + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override; + + void call(const boost::python::object& obj) const; + +private: + ContextPtr m_context; + std::shared_ptr m_partition; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_PARTITION_H diff --git a/SEPythonModule/SEPythonModule/PipelineReceiver.h b/SEPythonModule/SEPythonModule/PipelineReceiver.h new file mode 100644 index 000000000..9d26fea10 --- /dev/null +++ b/SEPythonModule/SEPythonModule/PipelineReceiver.h @@ -0,0 +1,91 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H +#define SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H + +#include "Pyston/GIL.h" +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEPythonModule/SourceInterface.h" +#include + +namespace SourceXPy { + +using SourceReceiverIfce = SourceXtractor::PipelineReceiver; +using GroupReceiverIfce = SourceXtractor::PipelineReceiver; + +class ProcessSourcesEvent { +public: + SourceXtractor::ProcessSourcesEvent m_event; + + std::string repr() const; +}; + +class AllFramesDone : public SourceXtractor::SelectionCriteria { +public: + bool mustBeProcessed(const SourceXtractor::SourceInterface& source) const override; +}; + +class PipelineSourceReceiver : public SourceReceiverIfce { +public: + explicit PipelineSourceReceiver(boost::python::object callback, ContextPtr context) + : m_callback(std::move(callback)), m_context(std::move(context)){}; + + ~PipelineSourceReceiver() override = default; + + void receiveSource(std::unique_ptr source) override { + Pyston::GILLocker gil; + m_callback(std::make_shared(m_context, std::move(source))); + } + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { + Pyston::GILLocker gil; + m_callback(ProcessSourcesEvent{event}); + } + +private: + boost::python::object m_callback; + ContextPtr m_context; +}; + +class PipelineGroupReceiver : public GroupReceiverIfce { +public: + explicit PipelineGroupReceiver(boost::python::object callback, ContextPtr context) + : m_callback(std::move(callback)), m_context(std::move(context)){}; + + ~PipelineGroupReceiver() override = default; + + void receiveSource(std::unique_ptr source) override { + Pyston::GILLocker gil; + m_callback(std::make_shared(SourceGroup{std::move(source), m_context})); + } + + void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { + Pyston::GILLocker gil; + m_callback(ProcessSourcesEvent{event}); + } + +private: + boost::python::object m_callback; + ContextPtr m_context; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H diff --git a/SEPythonModule/SEPythonModule/Segmentation.h b/SEPythonModule/SEPythonModule/Segmentation.h new file mode 100644 index 000000000..d86596773 --- /dev/null +++ b/SEPythonModule/SEPythonModule/Segmentation.h @@ -0,0 +1,52 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_SEGMENTATION_H +#define SOURCEXTRACTORPLUSPLUS_SEGMENTATION_H + +#include "SEFramework/Pipeline/PipelineStage.h" +#include "SEPythonModule/Context.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +// Forward declaration +namespace SourceXtractor { +class Segmentation; +} + +namespace SourceXPy { + +class Segmentation { +public: + explicit Segmentation(ContextPtr context); + + std::string repr() const; + + void setNextStage(const boost::python::object& callback); + + void call() const; + +private: + ContextPtr m_context; + std::shared_ptr m_segmentation; + std::shared_ptr m_next_stage; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_SEGMENTATION_H diff --git a/SEPythonModule/SEPythonModule/SourceInterface.h b/SEPythonModule/SEPythonModule/SourceInterface.h new file mode 100644 index 000000000..9645d4eaf --- /dev/null +++ b/SEPythonModule/SEPythonModule/SourceInterface.h @@ -0,0 +1,83 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef SOURCEXTRACTORPLUSPLUS_SOURCEINTERFACE_H +#define SOURCEXTRACTORPLUSPLUS_SOURCEINTERFACE_H + +#include "SEFramework/Source/SourceGroupInterface.h" +#include "SEFramework/Source/SourceInterface.h" +#include "SEPythonModule/Context.h" +#include +#include +#include + +namespace SourceXPy { + +struct DetachedSource { + boost::python::dict m_attributes; + + std::string repr() const; + boost::python::list attributes() const; + boost::python::object attribute(const std::string& key) const; +}; + +struct AttachedSource { + ContextPtr m_context; + SourceXtractor::SourceInterface* m_source_ptr = nullptr; + + boost::python::object attribute(const std::string& key) const; + DetachedSource detach() const; +}; + +struct OwnedSource : public AttachedSource { + OwnedSource(ContextPtr context, std::unique_ptr source) + : AttachedSource{std::move(context), source.get()}, m_owned_source(std::move(source)) {} + + std::string repr() const; + + std::unique_ptr m_owned_source; +}; + +struct EntangledSource : public AttachedSource { + explicit EntangledSource(ContextPtr context) : AttachedSource{std::move(context)} {}; + + std::string repr() const; +}; + +struct SourceGroup { + struct Iterator { + SourceXtractor::SourceGroupInterface::const_iterator m_i; + SourceXtractor::SourceGroupInterface::const_iterator m_end; + std::shared_ptr m_holder; + + // It always returns the same! + std::shared_ptr next(); + }; + + std::unique_ptr m_group; + ContextPtr m_context; + + std::string repr() const; + size_t size() const; + boost::python::object attribute(const std::string& key) const; + Iterator iter() const; +}; + +} // namespace SourceXPy + +#endif // SOURCEXTRACTORPLUSPLUS_SOURCEINTERFACE_H diff --git a/SEPythonModule/src/lib/ConfigAdapter.cpp b/SEPythonModule/src/lib/ConfigAdapter.cpp new file mode 100644 index 000000000..db8ad4ab8 --- /dev/null +++ b/SEPythonModule/src/lib/ConfigAdapter.cpp @@ -0,0 +1,62 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/ConfigAdapter.h" +#include +#include + +namespace SourceXPy { + +namespace py = boost::python; +namespace po = boost::program_options; + +ConfigAdapter::ConfigAdapter(po::options_description opt_desc) : m_option_descriptions(std::move(opt_desc)) { + // Populate defaults + for (const auto& p : m_option_descriptions.options()) { + boost::any default_value; + if (p->semantic()->apply_default(default_value)) { + m_options[p->long_name()] = boost::program_options::variable_value(default_value, true); + } + } +} + +void ConfigAdapter::fromPython(const py::dict& config) { + auto pairs = config.items(); + for (ssize_t i = 0; i < len(pairs); ++i) { + py::tuple pair(pairs[i]); + std::string key = py::extract(pair[0]); + // Dirty trick, serialize into string and let boost options parse back to whatever is expected + // This is necessary because we can not easily recover the actual expected type unless there is + // a default, and we want to work also in corner cases as when an integer-like or double is used + // to setup a float + std::string value = py::extract(pair[1].attr("__str__")()); + + const auto opt = m_option_descriptions.find_nothrow(key, false); + if (opt) { + boost::any opt_value; + opt->semantic()->parse(opt_value, {value}, true); + m_options[opt->long_name()] = boost::program_options::variable_value(opt_value, false); + } + } +} + +auto ConfigAdapter::getOptions() const -> const config_map_t& { + return m_options; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Context.cpp b/SEPythonModule/src/lib/Context.cpp new file mode 100644 index 000000000..acf81850e --- /dev/null +++ b/SEPythonModule/src/lib/Context.cpp @@ -0,0 +1,140 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Context.h" +#include "SEFramework/Output/OutputRegistry.h" +#include "SEFramework/Plugin/PluginManager.h" +#include "SEFramework/Source/SourceGroupWithOnDemandPropertiesFactory.h" +#include "SEFramework/Source/SourceWithOnDemandPropertiesFactory.h" +#include "SEFramework/Task/TaskFactoryRegistry.h" +#include "SEFramework/Task/TaskProvider.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" +#include "SEImplementation/Configuration/OutputConfig.h" +#include "SEImplementation/Deblending/DeblendingFactory.h" +#include "SEImplementation/Grouping/GroupingFactory.h" +#include "SEImplementation/Measurement/MeasurementFactory.h" +#include "SEImplementation/Output/OutputFactory.h" +#include "SEImplementation/Partition/PartitionFactory.h" +#include "SEImplementation/Property/RegisterConverters.h" +#include "SEImplementation/Segmentation/SegmentationFactory.h" +#include "SEMain/SourceXtractorConfig.h" +#include "SEPythonModule/ConfigAdapter.h" +#include +#include +#include + +namespace SourceXPy { + +namespace Configuration = Euclid::Configuration; +namespace py = boost::python; +namespace se = SourceXtractor; +using Euclid::Configuration::ConfigManager; + +thread_local std::shared_ptr Context::s_context = nullptr; + +Context::Context(const py::dict& global_config, const py::object& measurement_config) + : m_task_factory_registry(std::make_shared()) + , m_task_provider(std::make_shared(m_task_factory_registry)) + , m_output_registry(std::make_shared()) + , m_plugin_manager(std::make_shared(m_task_factory_registry, m_output_registry, "", + std::vector{})) + , m_source_factory(std::make_shared(m_task_provider)) + , m_group_factory(std::make_shared(m_task_provider)) + , m_segmentation_factory(std::make_shared(m_task_provider)) + , m_partition_factory(std::make_shared(m_source_factory)) + , m_grouping_factory(std::make_shared(m_group_factory)) + , m_deblending_factory(std::make_shared(m_source_factory)) + , m_measurement_factory(std::make_shared(m_output_registry)) + , m_output_factory(std::make_shared(m_output_registry)) { + + // This is *very* risky, but the only alternative to a big refactoring + m_config_manager.reset( + &ConfigManager::getInstance(Configuration::getUniqueManagerId()), + [](const ConfigManager* manager) { Euclid::Configuration::ConfigManager::deregisterInstance(manager->getId()); }); + + m_plugin_manager->loadPlugins(); + + // Register configuration + m_segmentation_factory->reportConfigDependencies(*m_config_manager); + m_partition_factory->reportConfigDependencies(*m_config_manager); + m_grouping_factory->reportConfigDependencies(*m_config_manager); + m_deblending_factory->reportConfigDependencies(*m_config_manager); + m_plugin_manager->getTaskFactoryRegistry().reportConfigDependencies(*m_config_manager); + m_measurement_factory->reportConfigDependencies(*m_config_manager); + m_output_factory->reportConfigDependencies(*m_config_manager); + + m_config_manager->registerConfiguration(); + m_config_manager->registerConfiguration(); + m_config_manager->registerConfiguration(); + + // Configure + ConfigAdapter config_wrapper(m_config_manager->closeRegistration()); + config_wrapper.fromPython(global_config); + auto options = config_wrapper.getOptions(); + + // Override the python object to use for the measurement configuration + options["python-config-object"].value() = boost::any(measurement_config); + m_config_manager->initialize(options); + + m_segmentation_factory->configure(*m_config_manager); + m_partition_factory->configure(*m_config_manager); + m_grouping_factory->configure(*m_config_manager); + m_deblending_factory->configure(*m_config_manager); + m_task_factory_registry->configure(*m_config_manager); + m_measurement_factory->configure(*m_config_manager); + m_output_factory->configure(*m_config_manager); + + // Register the output properties + m_task_factory_registry->registerPropertyInstances(*m_output_registry); + + // Get hold of the sources-to-row converter + auto enabled_properties = m_config_manager->getConfiguration().getOutputProperties(); + m_source_to_row = m_output_registry->getSourceToRowConverter(enabled_properties); + se::registerDefaultConverters(*m_output_registry); +} + +py::dict Context::get_properties() const { + py::dict properties; + const auto property_names = m_output_registry->getOutputPropertyNames(); + for (auto& prop_name : property_names) { + const auto column_names = m_output_registry->getColumns(prop_name); + py::list attr_names; + boost::for_each(column_names, [&attr_names](const std::string& c) { attr_names.append(c); }); + properties[prop_name] = attr_names; + } + return properties; +} + +void Context::enter(const std::shared_ptr& context) { + s_context = context; +} + +void Context::exit(const std::shared_ptr&, [[maybe_unused]] const py::object& exc_type, + [[maybe_unused]] const py::object& exc_value, [[maybe_unused]] const py::object& traceback) { + s_context.reset(); +} + +const std::shared_ptr& Context::get_global_context() { + if (!s_context) { + PyErr_SetString(PyExc_RuntimeError, "Need an explicit, or an active Context instance"); + py::throw_error_already_set(); + } + return s_context; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Deblending.cpp b/SEPythonModule/src/lib/Deblending.cpp new file mode 100644 index 000000000..28d077f00 --- /dev/null +++ b/SEPythonModule/src/lib/Deblending.cpp @@ -0,0 +1,70 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Deblending.h" +#include "Pyston/GIL.h" +#include "SEImplementation/Deblending/DeblendingFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; + +namespace SourceXPy { + +Deblending::Deblending(ContextPtr context) : m_context(std::move(context)) { + m_deblending = m_context->m_deblending_factory->createDeblending(); +} + +std::string Deblending::repr() const { + return "Deblending"; +} + +void Deblending::setNextStage(const py::object& callback) { + m_deblending->setNextStage(std::make_shared(callback, m_context)); +} + +void Deblending::receiveSource(std::unique_ptr source) { + m_deblending->receiveSource(std::move(source)); +} + +void Deblending::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_deblending->receiveProcessSignal(event); +} + +void Deblending::call(const py::object& obj) const { + py::extract group_wrapper(obj); + if (group_wrapper.check()) { + const auto& group_ptr = group_wrapper().m_group; + Pyston::SaveThread save_thread; + std::unique_ptr cloned_group_ptr( + dynamic_cast(group_ptr->clone().release())); + m_deblending->receiveSource(std::move(cloned_group_ptr)); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper().m_event; + Pyston::SaveThread save_thread; + m_deblending->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Deblending: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/FitsOutput.cpp b/SEPythonModule/src/lib/FitsOutput.cpp new file mode 100644 index 000000000..24f0708f6 --- /dev/null +++ b/SEPythonModule/src/lib/FitsOutput.cpp @@ -0,0 +1,77 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/FitsOutput.h" +#include "SEImplementation/Output/OutputFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include "SEPythonModule/SourceInterface.h" +#include + +namespace py = boost::python; + +namespace SourceXPy { + +FitsOutput::FitsOutput(SourceXPy::ContextPtr context) : m_context(std::move(context)) { + m_output = m_context->m_output_factory->createOutput(); +} + +FitsOutput::~FitsOutput() = default; + +std::string FitsOutput::repr() const { + return "FitsOutput"; +} + +void FitsOutput::receiveSource(std::unique_ptr group) { + m_output->receiveSource(std::move(group)); +} + +void FitsOutput::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_output->receiveProcessSignal(event); + auto all_done = std::dynamic_pointer_cast(event.m_selection_criteria); + if (all_done) { + m_semaphore.release(); + } +} + +void FitsOutput::call(const boost::python::object& obj) { + py::extract group(obj); + if (group.check()) { + const auto& group_ptr = group().m_group; + Pyston::SaveThread save_thread; + std::unique_ptr cloned_group_ptr( + dynamic_cast(group_ptr->clone().release())); + m_output->receiveSource(std::move(cloned_group_ptr)); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper().m_event; + Pyston::SaveThread save_thread; + m_output->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "FitsOutput: Unexpected python object received"); + py::throw_error_already_set(); +} + +void FitsOutput::get() { + Pyston::SaveThread save_thread; + m_semaphore.acquire(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Grouping.cpp b/SEPythonModule/src/lib/Grouping.cpp new file mode 100644 index 000000000..1178afeca --- /dev/null +++ b/SEPythonModule/src/lib/Grouping.cpp @@ -0,0 +1,72 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Grouping.h" +#include "SEImplementation/Grouping/GroupingFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; + +namespace SourceXPy { + +Grouping::Grouping(ContextPtr context) : m_context(std::move(context)) { + m_grouping = m_context->m_grouping_factory->createGrouping(); +} + +std::string Grouping::repr() const { + return "Grouping"; +} + +void Grouping::setNextStage(const py::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_grouping->setNextStage(cpp_ifce()); + } else { + m_grouping->setNextStage(std::make_shared(callback, m_context)); + } +} + +void Grouping::receiveSource(std::unique_ptr source) { + m_grouping->receiveSource(std::move(source)); +} + +void Grouping::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_grouping->receiveProcessSignal(event); +} + +void Grouping::call(const py::object& obj) const { + py::extract source_wrapper(obj); + if (source_wrapper.check()) { + const auto& source_ptr = source_wrapper().m_source_ptr; + Pyston::SaveThread save_thread; + m_grouping->receiveSource(source_ptr->clone()); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper().m_event; + Pyston::SaveThread save_thread; + m_grouping->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Grouping: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Measurement.cpp b/SEPythonModule/src/lib/Measurement.cpp new file mode 100644 index 000000000..1e2d042bc --- /dev/null +++ b/SEPythonModule/src/lib/Measurement.cpp @@ -0,0 +1,79 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Measurement.h" +#include "SEImplementation/Measurement/MeasurementFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; + +namespace SourceXPy { + +Measurement::Measurement(ContextPtr context) : m_context(std::move(context)) { + m_measurement = m_context->m_measurement_factory->getMeasurement(); +} + +Measurement::~Measurement() { + m_measurement->stopThreads(); +} + +std::string Measurement::repr() const { + return "Measurement"; +} + +void Measurement::setNextStage(const boost::python::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_measurement->setNextStage(cpp_ifce()); + } else { + m_measurement->setNextStage(std::make_shared(callback, m_context)); + } + m_measurement->startThreads(); +} + +void Measurement::receiveSource(std::unique_ptr group) { + m_measurement->receiveSource(std::move(group)); +} + +void Measurement::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_measurement->receiveProcessSignal(event); +} + +void Measurement::call(const py::object& obj) { + py::extract group(obj); + if (group.check()) { + const auto& group_ptr = group().m_group; + Pyston::SaveThread save_thread; + std::unique_ptr cloned_group_ptr( + dynamic_cast(group_ptr->clone().release())); + m_measurement->receiveSource(std::move(cloned_group_ptr)); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper().m_event; + Pyston::SaveThread save_thread; + m_measurement->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Measurement: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/NumpyOutput.cpp b/SEPythonModule/src/lib/NumpyOutput.cpp new file mode 100644 index 000000000..80cf9d68b --- /dev/null +++ b/SEPythonModule/src/lib/NumpyOutput.cpp @@ -0,0 +1,79 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/NumpyOutput.h" +#include "Configuration/ConfigManager.h" +#include "Pyston/Table2Numpy.h" +#include "SEImplementation/Measurement/MeasurementFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include "SEPythonModule/SourceInterface.h" +#include +#include + +using Euclid::Table::Row; +using Euclid::Table::Table; +namespace py = boost::python; +namespace se = SourceXtractor; + +namespace SourceXPy { + +NumpyOutput::NumpyOutput(ContextPtr context) : m_context(std::move(context)) { + m_source_to_row = m_context->m_source_to_row; +} + +NumpyOutput::~NumpyOutput() {} + +std::string NumpyOutput::repr() const { + return "Output"; +} + +void NumpyOutput::receiveSource(std::unique_ptr group) { + boost::transform(*group, std::back_inserter(m_rows), m_source_to_row); +} + +void NumpyOutput::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + auto all_done = std::dynamic_pointer_cast(event.m_selection_criteria); + if (all_done) { + m_semaphore.release(); + } +} + +void NumpyOutput::call(const py::object& obj) { + py::extract group(obj); + if (group.check()) { + const auto& group_ptr = group().m_group; + Pyston::SaveThread save_thread; + boost::transform(*group_ptr, std::back_inserter(m_rows), m_source_to_row); + return; + } + py::extract event_wrapper(obj); + if (!event_wrapper.check()) { + PyErr_SetString(PyExc_TypeError, "NumpyOutput: Unexpected python object received"); + py::throw_error_already_set(); + } +} + +py::object NumpyOutput::getTable() { + { + Pyston::SaveThread save_thread; + m_semaphore.acquire(); + } + return Pyston::table2numpy(Table(m_rows)); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Partition.cpp b/SEPythonModule/src/lib/Partition.cpp new file mode 100644 index 000000000..2341ea82d --- /dev/null +++ b/SEPythonModule/src/lib/Partition.cpp @@ -0,0 +1,72 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Partition.h" +#include "SEImplementation/Partition/PartitionFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include + +namespace py = boost::python; + +namespace SourceXPy { + +Partition::Partition(ContextPtr context) : m_context(std::move(context)) { + m_partition = m_context->m_partition_factory->getPartition(); +} + +std::string Partition::repr() const { + return "Partition"; +} + +void Partition::setNextStage(const py::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_partition->setNextStage(cpp_ifce()); + } else { + m_partition->setNextStage(std::make_shared(callback, m_context)); + } +} + +void Partition::receiveSource(std::unique_ptr source) { + m_partition->receiveSource(std::move(source)); +} + +void Partition::receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) { + m_partition->receiveProcessSignal(event); +} + +void Partition::call(const py::object& obj) const { + py::extract source_wrapper(obj); + if (source_wrapper.check()) { + const auto& source_ptr = source_wrapper().m_source_ptr; + Pyston::SaveThread save_thread; + m_partition->receiveSource(source_ptr->clone()); + return; + } + py::extract event_wrapper(obj); + if (event_wrapper.check()) { + const auto& event = event_wrapper().m_event; + Pyston::SaveThread save_thread; + m_partition->receiveProcessSignal(event); + return; + } + PyErr_SetString(PyExc_TypeError, "Partition: Unexpected python object received"); + py::throw_error_already_set(); +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp new file mode 100644 index 000000000..e63bd4bb0 --- /dev/null +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -0,0 +1,146 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Context.h" +#include "SEPythonModule/Deblending.h" +#include "SEPythonModule/FitsOutput.h" +#include "SEPythonModule/Grouping.h" +#include "SEPythonModule/Measurement.h" +#include "SEPythonModule/NumpyOutput.h" +#include "SEPythonModule/Partition.h" +#include "SEPythonModule/PipelineReceiver.h" +#include "SEPythonModule/Segmentation.h" +#include "SEPythonModule/SourceInterface.h" +#include +#include +#include +#include +#include +#include + +using namespace SourceXPy; +namespace py = boost::python; +namespace np = boost::python::numpy; + +namespace { +struct PickleDetachedSource : public py::pickle_suite { + static py::tuple getinitargs(const DetachedSource&) { + return py::make_tuple(); + } + + static py::tuple getstate(const DetachedSource& source) { + return py::make_tuple(source.m_attributes); + } + + static void setstate(DetachedSource& source, py::tuple state) { + source.m_attributes = py::extract(state[0]); + } +}; +} // namespace + +BOOST_PYTHON_MODULE(_SEPythonModule) { + np::initialize(); + py::object None; + + py::class_( + "Context", py::init((py::arg("config"), py::arg("measurement_config") = py::object()))) + .def("get_properties", &Context::get_properties) + .def("__enter__", &Context::enter) + .def("__exit__", &Context::exit); + py::register_ptr_to_python>(); + + py::class_("DetachedSource") + .def("__repr__", &DetachedSource::repr) + .def("__getattr__", &DetachedSource::attribute) + .def("__dir__", &DetachedSource::attributes) + .def_pickle(PickleDetachedSource()); + + py::class_("Source", py::no_init) + .def("__getattr__", &AttachedSource::attribute) + .def("detach", &AttachedSource::detach); + py::register_ptr_to_python>(); + + py::class_, boost::noncopyable>("OwnedSource", py::no_init) + .def("__repr__", &OwnedSource::repr); + py::register_ptr_to_python>(); + + py::class_, boost::noncopyable>("EntangledSource", py::no_init) + .def("__repr__", &EntangledSource::repr); + + py::register_ptr_to_python>(); + + py::class_("GroupIterator", py::no_init).def("__next__", &SourceGroup::Iterator::next); + + py::class_("Group", py::no_init) + .def("__repr__", &SourceGroup::repr) + .def("__getattr__", &SourceGroup::attribute) + .def("__len__", &SourceGroup::size) + .def("__iter__", &SourceGroup::iter); + py::register_ptr_to_python>(); + + py::class_("ProcessSourcesEvent", py::no_init).def("__repr__", &ProcessSourcesEvent::repr); + + py::class_("SourceReceiver", py::no_init); + py::class_("GroupReceiverIfce", py::no_init); + + py::class_("Segmentation", py::init>((py::arg("context") = None))) + .def("__repr__", &Segmentation::repr) + .def("set_next_stage", &Segmentation::setNextStage) + .def("__call__", &Segmentation::call); + + py::class_>("Partition", + py::init>((py::arg("context") = None))) + .def("__repr__", &Partition::repr) + .def("set_next_stage", &Partition::setNextStage) + .def("__call__", &Partition::call); + + py::class_>("Grouping", + py::init>((py::arg("context") = None))) + .def("__repr__", &Grouping::repr) + .def("set_next_stage", &Grouping::setNextStage) + .def("__call__", &Grouping::call); + + py::class_>("Deblending", + py::init>((py::arg("context") = None))) + .def("__repr__", &Deblending::repr) + .def("set_next_stage", &Deblending::setNextStage) + .def("__call__", &Deblending::call); + + py::class_, boost::noncopyable>( + "Measurement", py::init>((py::arg("context") = None))) + .def("__repr__", &Measurement::repr) + .def("set_next_stage", &Measurement::setNextStage) + .def("__call__", &Measurement::call); + + py::class_, boost::noncopyable>( + "NumpyOutput", py::init>((py::arg("Context") = None))) + .def("__repr__", &NumpyOutput::repr) + .def("__call__", &NumpyOutput::call) + .def("get", &NumpyOutput::getTable); + + py::class_, boost::noncopyable>( + "FitsOutput", py::init>((py::arg("Context") = None))) + .def("__repr__", &FitsOutput::repr) + .def("__call__", &FitsOutput::call) + .def("get", &FitsOutput::get); + + PyObject* pyston = PyInit_pyston(); + PyObject* modules = PyImport_GetModuleDict(); + PyDict_SetItemString(modules, "pyston", pyston); + Py_DECREF(pyston); +} diff --git a/SEPythonModule/src/lib/Segmentation.cpp b/SEPythonModule/src/lib/Segmentation.cpp new file mode 100644 index 000000000..788c7878e --- /dev/null +++ b/SEPythonModule/src/lib/Segmentation.cpp @@ -0,0 +1,78 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/Segmentation.h" +#include "Pyston/GIL.h" +#include "SEFramework/Pipeline/SourceGrouping.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" +#include "SEImplementation/Segmentation/SegmentationFactory.h" +#include "SEPythonModule/PipelineReceiver.h" +#include +#include + +namespace SourceXPy { + +namespace py = boost::python; +namespace se = SourceXtractor; + +using SourceXtractor::DetectionFrameConfig; +using SourceXtractor::DetectionImageFrame; +using SourceXtractor::SelectAllCriteria; + +Segmentation::Segmentation(ContextPtr context) : m_context(std::move(context)) { + m_segmentation = m_context->m_segmentation_factory->createSegmentation(); +} + +std::string Segmentation::repr() const { + return "Segmentation"; +} + +void Segmentation::setNextStage(const py::object& callback) { + py::extract> cpp_ifce(callback); + if (cpp_ifce.check()) { + m_next_stage = cpp_ifce(); + } else { + m_next_stage = std::make_shared(callback, m_context); + } + m_segmentation->setNextStage(m_next_stage); +} + +void Segmentation::call() const { + Pyston::SaveThread save_thread; + const auto& detection_config = m_context->m_config_manager->getConfiguration(); + const auto& frames = detection_config.getDetectionFrames(); + boost::for_each(frames, + [this](const std::shared_ptr& frame) { m_segmentation->processFrame(frame); }); + m_next_stage->receiveProcessSignal(se::ProcessSourcesEvent(std::make_shared())); +} + +std::string ProcessSourcesEvent::repr() const { + auto& ptr = m_event.m_selection_criteria; + if (std::dynamic_pointer_cast(ptr)) { + return "SelectAllCriteria"; + } else if (std::dynamic_pointer_cast(ptr)) { + return "AllFramesDone"; + } + return "LineSelectionCriteria"; +} + +bool AllFramesDone::mustBeProcessed(const SourceXtractor::SourceInterface&) const { + return true; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/SourceInterface.cpp b/SEPythonModule/src/lib/SourceInterface.cpp new file mode 100644 index 000000000..53a60ba2f --- /dev/null +++ b/SEPythonModule/src/lib/SourceInterface.cpp @@ -0,0 +1,192 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/SourceInterface.h" +#include "SEFramework/Output/OutputRegistry.h" +#include "SEFramework/Source/SourceFactory.h" +#include "SEFramework/Source/SourceGroupWithOnDemandPropertiesFactory.h" +#include "SEFramework/Source/SourceInterface.h" +#include "SEImplementation/Plugin/GroupInfo/GroupInfo.h" +#include "SEImplementation/Property/SourceId.h" +#include +#include +#include +#include +#include + +namespace SourceXPy { + +namespace py = boost::python; +namespace np = boost::python::numpy; +using Euclid::NdArray::NdArray; +using SourceXtractor::GroupInfo; +using SourceXtractor::PropertyId; +using SourceXtractor::SourceId; + +namespace { + +auto logger = Elements::Logging::getLogger("SePy"); + +struct CellToPythonVisitor : public boost::static_visitor { + + template + py::object operator()(const std::vector& vector, + std::enable_if_t::value>* = nullptr) const { + auto array = np::zeros(py::make_tuple(vector.size()), np::dtype::get_builtin()); + std::memcpy(array.get_data(), vector.data(), vector.size() * sizeof(T)); + return array; + } + + template + py::object operator()(const NdArray& nd) const { + py::list shape; + boost::for_each(nd.shape(), [&shape](size_t d) { shape.append(d); }); + auto array = np::zeros(py::tuple(shape), np::dtype::get_builtin()); + std::memcpy(array.get_data(), &(*nd.begin()), nd.size() * sizeof(T)); + return array; + } + + py::object operator()(const std::vector& vector) const { + auto array = np::zeros(py::make_tuple(vector.size()), np::dtype::get_builtin()); + size_t i = 0; + for (auto&& v : vector) { + array[i] = v; + ++i; + } + return array; + } + + py::object operator()(const std::string& v) const { + return py::object(v); + } + + template + py::object operator()(From&& v, typename std::enable_if_t>::value>* = nullptr) const { + return py::object(v); + } +}; + +} // namespace + +std::string DetachedSource::repr() const { + unsigned int source_id = py::extract(m_attributes["source_id"]); + return std::to_string(source_id) + " [DETACHED]"; +} + +py::list DetachedSource::attributes() const { + return m_attributes.keys(); +} + +py::object DetachedSource::attribute(const std::string& key) const { + return m_attributes[key]; +} + +py::object AttachedSource::attribute(const std::string& key) const { + if (!m_source_ptr) { + PyErr_SetString(PyExc_ReferenceError, "Not owned source is gone"); + throw py::error_already_set(); + } + // Trigger computation if needed + try { + auto property_type_index = m_context->m_output_registry->getPropertyForColumn(key); + PropertyId property_id(property_type_index, 0); + m_source_ptr->getProperty(property_id); + } catch (const std::out_of_range&) { + std::stringstream err_str("Source has no attribute "); + err_str << key; + PyErr_SetString(PyExc_AttributeError, err_str.str().c_str()); + py::throw_error_already_set(); + } + // Convert + const auto& converter = m_context->m_output_registry->getColumnConverter(key); + return boost::apply_visitor(CellToPythonVisitor(), converter.second(*m_source_ptr)); +} + +DetachedSource AttachedSource::detach() const { + using SourceXtractor::Property; + using SourceXtractor::PropertyId; + + if (!m_source_ptr) { + PyErr_SetString(PyExc_ReferenceError, "Not owned source is gone"); + py::throw_error_already_set(); + } + + DetachedSource detached; + const auto& output_registry = m_context->m_output_registry; + m_source_ptr->visitProperties([&detached, source_ptr = m_source_ptr, + output_registry](const PropertyId& prop_id, const std::shared_ptr&) { + auto columns = output_registry->getColumns(prop_id); + if (columns.empty()) { + logger.debug() << "C++ property serialization not found for " << prop_id.getString(); + } + for (const auto& column : columns) { + if (detached.m_attributes.contains(column)) { + continue; + } + const auto& converter = output_registry->getColumnConverter(column); + detached.m_attributes[column] = boost::apply_visitor(CellToPythonVisitor(), converter.second(*source_ptr)); + } + }); + return detached; +} + +std::string OwnedSource::repr() const { + auto source_id = m_source_ptr->getProperty().getSourceId(); + return std::to_string(source_id) + " [OWNED]"; +} + +std::string EntangledSource::repr() const { + if (!m_source_ptr) { + PyErr_SetString(PyExc_ReferenceError, + "Not owned entangled source is gone. Use its detach method if you need a copy."); + throw py::error_already_set(); + } + auto source_id = m_source_ptr->getProperty().getSourceId(); + return std::to_string(source_id) + " [ENTANGLED]"; +} + +std::string SourceGroup::repr() const { + auto& group_id = m_group->getProperty(); + return std::to_string(group_id.getGroupId()); +} + +size_t SourceGroup::size() const { + return m_group->size(); +} + +py::object SourceGroup::attribute([[maybe_unused]] const std::string& key) const { + return py::object(); +} + +SourceGroup::Iterator SourceGroup::iter() const { + return Iterator{m_group->begin(), m_group->end(), std::make_shared(m_context)}; +} + +std::shared_ptr SourceGroup::Iterator::next() { + if (m_i == m_end) { + // If anyone is still using it, it will fail on access + m_holder->m_source_ptr = nullptr; + PyErr_SetNone(PyExc_StopIteration); + py::throw_error_already_set(); + } + m_holder->m_source_ptr = &m_i->getRef(); + ++m_i; + return m_holder; +} + +} // namespace SourceXPy From 2a40f5671b0e89231fd1ed269f0c3bc11f449734 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Tue, 20 Sep 2022 13:45:34 +0200 Subject: [PATCH 14/35] Add a Python demo --- SEPythonWrapper/CMakeLists.txt | 2 + SEPythonWrapper/conf/SourceXtractorDemo.conf | 34 ++++ .../python/sourcextractor/__init__.py | 3 +- .../python/sourcextractor/pipeline.py | 53 +++++++ SEPythonWrapper/scripts/SourceXtractorDemo.py | 149 ++++++++++++++++++ 5 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 SEPythonWrapper/conf/SourceXtractorDemo.conf create mode 100644 SEPythonWrapper/python/sourcextractor/pipeline.py create mode 100644 SEPythonWrapper/scripts/SourceXtractorDemo.py diff --git a/SEPythonWrapper/CMakeLists.txt b/SEPythonWrapper/CMakeLists.txt index bcef6129e..183a75c15 100644 --- a/SEPythonWrapper/CMakeLists.txt +++ b/SEPythonWrapper/CMakeLists.txt @@ -65,9 +65,11 @@ elements_depends_on_subdirs(SEImplementation) # elements_install_scripts() #=============================================================================== elements_install_python_modules() +elements_install_scripts() #=============================================================================== # Add the elements_install_conf_files macro # Examples: # elements_install_conf_files() #=============================================================================== +elements_install_conf_files() diff --git a/SEPythonWrapper/conf/SourceXtractorDemo.conf b/SEPythonWrapper/conf/SourceXtractorDemo.conf new file mode 100644 index 000000000..486e0c264 --- /dev/null +++ b/SEPythonWrapper/conf/SourceXtractorDemo.conf @@ -0,0 +1,34 @@ +auto-kron-factor = 2.5 +auto-kron-min-radius = 3.5 +background-cell-size = 64 +smoothing-box-size = 3 +detection-threshold = 1.4953 +segmentation-algorithm = LUTZ +segmentation-use-filtering = true +segmentation-filter = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/default.conv +detection-image-interpolation = 1 +detection-image-interpolation-gap = 5 +use-cleaning = false +cleaning-minimum-area = 10 +detection-minimum-area = 5 +grouping-algorithm = SPLIT +magnitude-zero-point = 32.19 +tile-memory-limit = 512 +tile-size = 256 +model-fitting-iterations = 1000 +thread-count = 4 +partition-multithreshold = true +partition-threshold-count = 32 +partition-minimum-area = 3 +partition-minimum-contrast = 0.005 +psf-fwhm = 3 +psf-pixel-sampling = 1 +weight-use-symmetry = 1 +output-properties = SourceIDs,PixelCentroid,WorldCentroid,IsophotalFlux,SourceFlags +detection-image = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/img/sim12.fits.gz +weight-image = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/img/sim12.weight.fits.gz +weight-type = weight +weight-absolute = True +python-config-file = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/sim12_multi_modelfitting.py +# Custom filter! +snr = 10 diff --git a/SEPythonWrapper/python/sourcextractor/__init__.py b/SEPythonWrapper/python/sourcextractor/__init__.py index 16a48b8d4..ea8c771e9 100644 --- a/SEPythonWrapper/python/sourcextractor/__init__.py +++ b/SEPythonWrapper/python/sourcextractor/__init__.py @@ -15,5 +15,6 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA -from _SEPythonConfig import Flags +from _SEPythonConfig import Flags +from SOURCEXTRACTORPLUSPLUS_VERSION import SOURCEXTRACTORPLUSPLUS_VERSION_STRING as __version__ diff --git a/SEPythonWrapper/python/sourcextractor/pipeline.py b/SEPythonWrapper/python/sourcextractor/pipeline.py new file mode 100644 index 000000000..71d9304e2 --- /dev/null +++ b/SEPythonWrapper/python/sourcextractor/pipeline.py @@ -0,0 +1,53 @@ +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +from _SEPythonModule import * + + +class Pipeline: + """ + Wrap a set of pipeline stages and chains them + + :param stages: + List of pipeline stages + """ + + def __init__(self, stages): + if len(stages) < 1: + raise ValueError('Expecting at least one stage') + self.__first = stages[0] + self.__last = stages[-1] + for a, b in zip(stages[:-1], stages[1:]): + a.set_next_stage(b) + + def __call__(self): + """ + Trigger the execution of the pipeline + :return: + The last chain of the pipeline + """ + self.__first() + return self.__last + + +class DefaultPipeline(Pipeline): + """ + Default implementation of the sourcextractor++ pipeline, equivalent to running + the CLI manually + """ + + def __init__(self): + super().__init__([Segmentation(), Partition(), Grouping(), Deblending(), Measurement(), FitsOutput()]) diff --git a/SEPythonWrapper/scripts/SourceXtractorDemo.py b/SEPythonWrapper/scripts/SourceXtractorDemo.py new file mode 100644 index 000000000..977360436 --- /dev/null +++ b/SEPythonWrapper/scripts/SourceXtractorDemo.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import itertools +import os.path +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict + +import h5py +import numpy as np +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + + +class SNRFilter: + """ + Drop sources with a signal-to-noise below the configured limit. + It expect sources, so it must be inserted into the pipeline *before* the partitioning + + :param snr: float + Signal-to-noise ratio cut + """ + + def __init__(self, snr: float): + self.__snr = snr + self.__next = None + self.__dropped = [] + + @property + def dropped(self): + return self.__dropped + + def set_next_stage(self, stage): + self.__next = stage + + def __call__(self, obj): + """ + Apply the SNR filter + """ + if isinstance(obj, pipeline.Source): + if obj.isophotal_flux / obj.isophotal_flux_err < self.__snr: + self.__dropped.append(obj) + return + self.__next(obj) + + +class StoreStamps: + """ + Store the detection stamps into the HDF5 file + + :param hd5: h5py.File + Output HDF5 file + """ + + def __init__(self, hd5: h5py.File): + self.__hd5 = hd5 + self.__next = None + + def set_next_stage(self, stage): + self.__next = stage + + def __store_stamp(self, source): + stamp = source.detection_filtered_stamp + dataset = self.__hd5.create_dataset(f'sources/{source.source_id}', data=stamp) + dataset.attrs.create('CLASS', 'IMAGE', dtype='S6') + dataset.attrs.create('IMAGE_VERSION', '1.2', dtype='S4') + dataset.attrs.create('IMAGE_SUBCLASS', 'IMAGE_GRAYSCALE', dtype='S16') + dataset.attrs.create('IMAGE_MINMAXRANGE', [np.min(stamp), np.max(stamp)]) + + def __call__(self, obj): + """ + Supports being called with a single Source, or with a Group of sources + """ + match type(obj): + case pipeline.Source: + self.__store_stamp(obj) + case pipeline.Group: + [self.__store_stamp(source) for source in obj] + case pipeline.ProcessSourcesEvent: + pass + case _: + print(f'Unknown {type(obj)}') + self.__next(obj) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str, stamps: bool): + """ + Setup the sourcextractor++ pipeline, run it, and write the output to an HDF5 file + """ + h5 = None + if output_path.endswith('.h5'): + h5 = h5py.File(output_path, 'w') + Output = pipeline.NumpyOutput + else: + Output = pipeline.FitsOutput + config['output-catalog-filename'] = output_path + + snr_filter = SNRFilter(float(config.pop('snr', 5))) + with pipeline.Context(config): + stages = [pipeline.Segmentation(), pipeline.Partition(), snr_filter, pipeline.Grouping(), pipeline.Deblending()] + if stamps and h5 is not None: + stages.append(StoreStamps(h5)) + stages.extend([pipeline.Measurement(), Output()]) + pipe = pipeline.Pipeline(stages) + result = pipe().get() + print(f'Dropped {len(snr_filter.dropped)} sources') + + if h5 is not None: + h5.create_dataset(os.path.basename(config['detection-image']), data=result) + h5.close() + print(f'{output_path} created!') + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +# Entry point +if __name__ == '__main__': + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='HDF5', default='output.h5', help='Output file') + parser.add_argument('--with-stamps', action='store_true', default=False, help='Store source stamps') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file, args.with_stamps) From 55234d7ae93375f7a77f46fad48c879ce9163da4 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 22 Sep 2022 10:31:33 +0200 Subject: [PATCH 15/35] Fix build in centos7 --- SEPythonModule/SEPythonModule/SourceInterface.h | 7 +++++-- SEPythonModule/src/lib/SePyMain.cpp | 7 ++++++- SEPythonModule/src/lib/SourceInterface.cpp | 6 ++++-- SEPythonWrapper/python/sourcextractor/pipeline.py | 1 + SEPythonWrapper/scripts/SourceXtractorDemo.py | 1 + 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/SEPythonModule/SEPythonModule/SourceInterface.h b/SEPythonModule/SEPythonModule/SourceInterface.h index 9645d4eaf..78f85be53 100644 --- a/SEPythonModule/SEPythonModule/SourceInterface.h +++ b/SEPythonModule/SEPythonModule/SourceInterface.h @@ -40,13 +40,16 @@ struct AttachedSource { ContextPtr m_context; SourceXtractor::SourceInterface* m_source_ptr = nullptr; + AttachedSource(ContextPtr context, SourceXtractor::SourceInterface* source_ptr) + : m_context(std::move(context)), m_source_ptr(source_ptr) {} + boost::python::object attribute(const std::string& key) const; DetachedSource detach() const; }; struct OwnedSource : public AttachedSource { OwnedSource(ContextPtr context, std::unique_ptr source) - : AttachedSource{std::move(context), source.get()}, m_owned_source(std::move(source)) {} + : AttachedSource(std::move(context), source.get()), m_owned_source(std::move(source)) {} std::string repr() const; @@ -54,7 +57,7 @@ struct OwnedSource : public AttachedSource { }; struct EntangledSource : public AttachedSource { - explicit EntangledSource(ContextPtr context) : AttachedSource{std::move(context)} {}; + explicit EntangledSource(ContextPtr context) : AttachedSource(std::move(context), nullptr) {} std::string repr() const; }; diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index e63bd4bb0..2fc699d22 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -139,7 +139,12 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { .def("__call__", &FitsOutput::call) .def("get", &FitsOutput::get); - PyObject* pyston = PyInit_pyston(); +#if PY_MAJOR_VERSION >= 3 + PyObject* pyston = PyInit_pyston(); +#else + initpyston(); + PyObject* pyston = PyImport_ImportModule("_SEPythonConfig"); +#endif PyObject* modules = PyImport_GetModuleDict(); PyDict_SetItemString(modules, "pyston", pyston); Py_DECREF(pyston); diff --git a/SEPythonModule/src/lib/SourceInterface.cpp b/SEPythonModule/src/lib/SourceInterface.cpp index 53a60ba2f..3ec051af4 100644 --- a/SEPythonModule/src/lib/SourceInterface.cpp +++ b/SEPythonModule/src/lib/SourceInterface.cpp @@ -46,7 +46,7 @@ struct CellToPythonVisitor : public boost::static_visitor { template py::object operator()(const std::vector& vector, - std::enable_if_t::value>* = nullptr) const { + typename std::enable_if::value>::type* = nullptr) const { auto array = np::zeros(py::make_tuple(vector.size()), np::dtype::get_builtin()); std::memcpy(array.get_data(), vector.data(), vector.size() * sizeof(T)); return array; @@ -76,7 +76,9 @@ struct CellToPythonVisitor : public boost::static_visitor { } template - py::object operator()(From&& v, typename std::enable_if_t>::value>* = nullptr) const { + py::object + operator()(From&& v, + typename std::enable_if::type>::value>::type* = nullptr) const { return py::object(v); } }; diff --git a/SEPythonWrapper/python/sourcextractor/pipeline.py b/SEPythonWrapper/python/sourcextractor/pipeline.py index 71d9304e2..f92906a16 100644 --- a/SEPythonWrapper/python/sourcextractor/pipeline.py +++ b/SEPythonWrapper/python/sourcextractor/pipeline.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université # # This library is free software; you can redistribute it and/or modify it under diff --git a/SEPythonWrapper/scripts/SourceXtractorDemo.py b/SEPythonWrapper/scripts/SourceXtractorDemo.py index 977360436..4bc8ed6af 100644 --- a/SEPythonWrapper/scripts/SourceXtractorDemo.py +++ b/SEPythonWrapper/scripts/SourceXtractorDemo.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# -*- coding: utf-8 -*- # # Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université # From be8eb467eed844739ba4d12c81d64438e223ea72 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 10:44:07 +0200 Subject: [PATCH 16/35] Demo python 3.8 compatible --- SEPythonWrapper/scripts/SourceXtractorDemo.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/SEPythonWrapper/scripts/SourceXtractorDemo.py b/SEPythonWrapper/scripts/SourceXtractorDemo.py index 4bc8ed6af..72bb60ad5 100644 --- a/SEPythonWrapper/scripts/SourceXtractorDemo.py +++ b/SEPythonWrapper/scripts/SourceXtractorDemo.py @@ -87,15 +87,12 @@ def __call__(self, obj): """ Supports being called with a single Source, or with a Group of sources """ - match type(obj): - case pipeline.Source: - self.__store_stamp(obj) - case pipeline.Group: - [self.__store_stamp(source) for source in obj] - case pipeline.ProcessSourcesEvent: - pass - case _: - print(f'Unknown {type(obj)}') + if isinstance(obj, pipeline.Source): + self.__store_stamp(obj) + elif isinstance(obj, pipeline.Group): + [self.__store_stamp(source) for source in obj] + elif not isinstance(obj, pipeline.ProcessSourcesEvent): + print(f'Unknown {type(obj)}') self.__next(obj) From 6ff371744c4d772f911f46c5cab46a8ec9b1825b Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 11:03:28 +0200 Subject: [PATCH 17/35] Keep a reference to the thread pool --- SEPythonModule/SEPythonModule/Context.h | 2 ++ SEPythonModule/src/lib/Context.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/SEPythonModule/SEPythonModule/Context.h b/SEPythonModule/SEPythonModule/Context.h index db2ce9fa6..548195c15 100644 --- a/SEPythonModule/SEPythonModule/Context.h +++ b/SEPythonModule/SEPythonModule/Context.h @@ -19,6 +19,7 @@ #ifndef SOURCEXTRACTORPLUSPLUS_CONTEXT_H #define SOURCEXTRACTORPLUSPLUS_CONTEXT_H +#include #include
#include #include @@ -73,6 +74,7 @@ class Context { std::shared_ptr m_measurement_factory; std::shared_ptr m_output_factory; std::function m_source_to_row; + std::shared_ptr m_thread_pool; static void enter(const std::shared_ptr&); static void exit(const std::shared_ptr&, const boost::python::object& exc_type, diff --git a/SEPythonModule/src/lib/Context.cpp b/SEPythonModule/src/lib/Context.cpp index acf81850e..55b672812 100644 --- a/SEPythonModule/src/lib/Context.cpp +++ b/SEPythonModule/src/lib/Context.cpp @@ -24,6 +24,7 @@ #include "SEFramework/Task/TaskFactoryRegistry.h" #include "SEFramework/Task/TaskProvider.h" #include "SEImplementation/Configuration/DetectionFrameConfig.h" +#include "SEImplementation/Configuration/MultiThreadingConfig.h" #include "SEImplementation/Configuration/OutputConfig.h" #include "SEImplementation/Deblending/DeblendingFactory.h" #include "SEImplementation/Grouping/GroupingFactory.h" @@ -99,6 +100,9 @@ Context::Context(const py::dict& global_config, const py::object& measurement_co m_measurement_factory->configure(*m_config_manager); m_output_factory->configure(*m_config_manager); + // Get the thread pool + m_thread_pool = m_config_manager->getConfiguration().getThreadPool(); + // Register the output properties m_task_factory_registry->registerPropertyInstances(*m_output_registry); From e9cd46a160e3fa55971fc96cf70e518dd650bbf3 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 11:04:47 +0200 Subject: [PATCH 18/35] Translate Python exception to Elements::Exception --- .../SEPythonModule/PipelineReceiver.h | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/SEPythonModule/SEPythonModule/PipelineReceiver.h b/SEPythonModule/SEPythonModule/PipelineReceiver.h index 9d26fea10..566766df7 100644 --- a/SEPythonModule/SEPythonModule/PipelineReceiver.h +++ b/SEPythonModule/SEPythonModule/PipelineReceiver.h @@ -19,10 +19,11 @@ #ifndef SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H #define SOURCEXTRACTORPLUSPLUS_PIPELINERECEIVER_H -#include "Pyston/GIL.h" #include "SEFramework/Pipeline/PipelineStage.h" #include "SEFramework/Pipeline/SourceGrouping.h" #include "SEPythonModule/SourceInterface.h" +#include +#include #include namespace SourceXPy { @@ -51,12 +52,20 @@ class PipelineSourceReceiver : public SourceReceiverIfce { void receiveSource(std::unique_ptr source) override { Pyston::GILLocker gil; - m_callback(std::make_shared(m_context, std::move(source))); + try { + m_callback(std::make_shared(m_context, std::move(source))); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } } void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { Pyston::GILLocker gil; - m_callback(ProcessSourcesEvent{event}); + try { + m_callback(ProcessSourcesEvent{event}); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } } private: @@ -73,12 +82,20 @@ class PipelineGroupReceiver : public GroupReceiverIfce { void receiveSource(std::unique_ptr source) override { Pyston::GILLocker gil; - m_callback(std::make_shared(SourceGroup{std::move(source), m_context})); + try { + m_callback(std::make_shared(SourceGroup{std::move(source), m_context})); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } } void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { Pyston::GILLocker gil; - m_callback(ProcessSourcesEvent{event}); + try { + m_callback(ProcessSourcesEvent{event}); + } catch (const boost::python::error_already_set&) { + throw Pyston::Exception(); + } } private: From b113a38b8956fef0af574ecb2be751ac57569301 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 12:08:59 +0200 Subject: [PATCH 19/35] Flush events following the received order --- .../Measurement/MultithreadedMeasurement.h | 4 +++- .../Measurement/MultithreadedMeasurement.cpp | 20 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h index 7e46a1390..7e0c8d6c2 100644 --- a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -68,6 +69,7 @@ class MultithreadedMeasurement : public Measurement { std::condition_variable m_new_output; std::list>> m_output_queue; + std::list> m_event_queue; std::mutex m_output_queue_mutex; Euclid::Semaphore m_semaphore; }; diff --git a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp index b29fdecff..6e0569105 100644 --- a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp +++ b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -127,17 +128,26 @@ void MultithreadedMeasurement::outputThreadLoop() { // Process the output queue while (!m_output_queue.empty()) { - sendSource(std::move(m_output_queue.front().second)); + auto next_source = std::move(m_output_queue.front()); m_output_queue.pop_front(); + + // Flush events received before this source + while (!m_event_queue.empty() && m_event_queue.front().first <= next_source.first) { + sendProcessSignal(m_event_queue.front().second); + m_event_queue.pop_front(); + } + + // Flush source + sendSource(std::move(next_source.second)); } - if (m_input_done && m_thread_pool->running() + m_thread_pool->queued() == 0 && - m_output_queue.empty()) { + if (m_input_done && m_thread_pool->running() + m_thread_pool->queued() == 0 && m_output_queue.empty()) { break; } } } void MultithreadedMeasurement::receiveProcessSignal(const ProcessSourcesEvent& event) { - sendProcessSignal(event); + std::unique_lock output_lock(m_output_queue_mutex); + m_event_queue.emplace_back(m_group_counter, event); } From 908ea614423b775cf7ff19a5c494df00cbb0d3d2 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 12:09:43 +0200 Subject: [PATCH 20/35] Properly catch exceptions during measurement --- SEPythonModule/SEPythonModule/FitsOutput.h | 2 +- SEPythonModule/SEPythonModule/NumpyOutput.h | 2 +- SEPythonModule/src/lib/FitsOutput.cpp | 15 ++++-- SEPythonModule/src/lib/Measurement.cpp | 6 ++- SEPythonModule/src/lib/NumpyOutput.cpp | 13 ++++- SEPythonModule/src/lib/SePyMain.cpp | 60 ++++++++++++++++++++- 6 files changed, 88 insertions(+), 10 deletions(-) diff --git a/SEPythonModule/SEPythonModule/FitsOutput.h b/SEPythonModule/SEPythonModule/FitsOutput.h index 9e3655a32..f1f90d4d6 100644 --- a/SEPythonModule/SEPythonModule/FitsOutput.h +++ b/SEPythonModule/SEPythonModule/FitsOutput.h @@ -43,7 +43,7 @@ class FitsOutput : public SourceXtractor::PipelineReceiverm_thread_pool->checkForException(true); + timeout -= try_wait; + if (timeout <= std::chrono::microseconds::zero()) { + PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); + py::throw_error_already_set(); + } + } + m_context->m_thread_pool->checkForException(true); } } // namespace SourceXPy diff --git a/SEPythonModule/src/lib/Measurement.cpp b/SEPythonModule/src/lib/Measurement.cpp index 1e2d042bc..e0272a33d 100644 --- a/SEPythonModule/src/lib/Measurement.cpp +++ b/SEPythonModule/src/lib/Measurement.cpp @@ -30,7 +30,11 @@ Measurement::Measurement(ContextPtr context) : m_context(std::move(context)) { } Measurement::~Measurement() { - m_measurement->stopThreads(); + try { + m_measurement->stopThreads(); + } catch (...) { + // Must not throw from a destructor + } } std::string Measurement::repr() const { diff --git a/SEPythonModule/src/lib/NumpyOutput.cpp b/SEPythonModule/src/lib/NumpyOutput.cpp index 80cf9d68b..6a38f14ca 100644 --- a/SEPythonModule/src/lib/NumpyOutput.cpp +++ b/SEPythonModule/src/lib/NumpyOutput.cpp @@ -68,11 +68,20 @@ void NumpyOutput::call(const py::object& obj) { } } -py::object NumpyOutput::getTable() { +py::object NumpyOutput::getTable(std::chrono::microseconds timeout) { + static constexpr std::chrono::seconds try_wait(1); { Pyston::SaveThread save_thread; - m_semaphore.acquire(); + while (!m_semaphore.try_acquire_for(try_wait)) { + m_context->m_thread_pool->checkForException(true); + timeout -= try_wait; + if (timeout <= std::chrono::microseconds::zero()) { + PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); + py::throw_error_already_set(); + } + } } + m_context->m_thread_pool->checkForException(true); return Pyston::table2numpy(Table(m_rows)); } diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index 2fc699d22..7defbfde1 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -32,12 +32,17 @@ #include #include #include +#include using namespace SourceXPy; namespace py = boost::python; namespace np = boost::python::numpy; namespace { + +/** + * Pickle detached sources + */ struct PickleDetachedSource : public py::pickle_suite { static py::tuple getinitargs(const DetachedSource&) { return py::make_tuple(); @@ -51,12 +56,63 @@ struct PickleDetachedSource : public py::pickle_suite { source.m_attributes = py::extract(state[0]); } }; + +/** + * Convert std::chrono::duration to datetime.timedelta + */ +struct timedelta_from_std_duration { + timedelta_from_std_duration() { + py::to_python_converter(); + } + + static PyObject* convert(std::chrono::microseconds us) { + auto seconds = std::chrono::duration_cast(us); + us -= seconds; + auto hours = std::chrono::duration_cast(seconds); + seconds -= hours; + return PyDelta_FromDSU(hours.count() / 24, seconds.count(), us.count()); + } +}; + +/** + * Convert datetime.timedelta to std::chrono::duration + */ +struct std_duration_from_timedelta { + std_duration_from_timedelta() { + py::converter::registry::push_back(&convertible, &construct, py::type_id()); + } + + static void* convertible(PyObject* obj_ptr) { + if (PyDelta_Check(obj_ptr)) { + return obj_ptr; + } + return nullptr; + } + + static void construct(PyObject* obj_ptr, py::converter::rvalue_from_python_stage1_data* data) { + auto timedelta = reinterpret_cast(obj_ptr); + auto days = std::chrono::hours(PyDateTime_DELTA_GET_DAYS(timedelta) * 24); + auto seconds = std::chrono::seconds(PyDateTime_DELTA_GET_SECONDS(timedelta)); + auto microseconds = std::chrono::microseconds(PyDateTime_DELTA_GET_MICROSECONDS(timedelta)); + auto duration = days + seconds + microseconds; + + auto storage = + (reinterpret_cast*>(data))->storage.bytes; + new (storage) std::chrono::microseconds(duration); + data->convertible = storage; + } +}; + } // namespace BOOST_PYTHON_MODULE(_SEPythonModule) { np::initialize(); py::object None; + PyDateTime_IMPORT; + std_duration_from_timedelta(); + timedelta_from_std_duration(); + py::class_( "Context", py::init((py::arg("config"), py::arg("measurement_config") = py::object()))) .def("get_properties", &Context::get_properties) @@ -131,13 +187,13 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { "NumpyOutput", py::init>((py::arg("Context") = None))) .def("__repr__", &NumpyOutput::repr) .def("__call__", &NumpyOutput::call) - .def("get", &NumpyOutput::getTable); + .def("get", &NumpyOutput::getTable, (py::arg("timeout") = std::chrono::microseconds::max())); py::class_, boost::noncopyable>( "FitsOutput", py::init>((py::arg("Context") = None))) .def("__repr__", &FitsOutput::repr) .def("__call__", &FitsOutput::call) - .def("get", &FitsOutput::get); + .def("get", &FitsOutput::get, (py::arg("timeout") = std::chrono::microseconds::max())); #if PY_MAJOR_VERSION >= 3 PyObject* pyston = PyInit_pyston(); From 36dd2cadacbb35fd0df1d5350a56ef83b9e3bcfa Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 13:15:55 +0200 Subject: [PATCH 21/35] MultithreadedMeasurement must send events in order --- .../Measurement/MultithreadedMeasurement.h | 47 ++++++++-------- .../Measurement/MultithreadedMeasurement.cpp | 53 +++++++++---------- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h index 7e0c8d6c2..e7817fbb2 100644 --- a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h @@ -25,27 +25,28 @@ #ifndef _SEIMPLEMENTATION_OUTPUT_MULTITHREADEDMEASUREMENT_H_ #define _SEIMPLEMENTATION_OUTPUT_MULTITHREADEDMEASUREMENT_H_ -#include -#include -#include -#include -#include -#include "AlexandriaKernel/ThreadPool.h" #include "AlexandriaKernel/Semaphore.h" +#include "AlexandriaKernel/ThreadPool.h" #include "SEFramework/Pipeline/Measurement.h" +#include +#include +#include +#include +#include namespace SourceXtractor { class MultithreadedMeasurement : public Measurement { public: - using SourceToRowConverter = std::function; MultithreadedMeasurement(SourceToRowConverter source_to_row, const std::shared_ptr& thread_pool, unsigned max_queue_size) - : m_source_to_row(source_to_row), - m_thread_pool(thread_pool), - m_group_counter(0), - m_input_done(false), m_abort_raised(false), m_semaphore(max_queue_size) {} + : m_source_to_row(source_to_row) + , m_thread_pool(thread_pool) + , m_group_counter(0) + , m_input_done(false) + , m_abort_raised(false) + , m_semaphore(max_queue_size) {} ~MultithreadedMeasurement() override; @@ -57,23 +58,27 @@ class MultithreadedMeasurement : public Measurement { void synchronizeThreads() override; private: + using QueuePair = std::pair>; + // We want O(1) for the *lowest* value (received order) + using OutputQueue = std::priority_queue, std::greater<>>; + static void outputThreadStatic(MultithreadedMeasurement* measurement); - void outputThreadLoop(); + void outputThreadLoop(); - SourceToRowConverter m_source_to_row; + SourceToRowConverter m_source_to_row; std::shared_ptr m_thread_pool; - std::unique_ptr m_output_thread; + std::unique_ptr m_output_thread; - int m_group_counter; + int m_group_counter; std::atomic_bool m_input_done, m_abort_raised; - std::condition_variable m_new_output; - std::list>> m_output_queue; - std::list> m_event_queue; - std::mutex m_output_queue_mutex; - Euclid::Semaphore m_semaphore; + std::condition_variable m_new_output; + OutputQueue m_output_queue; + std::queue> m_event_queue; + std::mutex m_output_queue_mutex; + Euclid::Semaphore m_semaphore; }; -} +} // namespace SourceXtractor #endif /* _SEIMPLEMENTATION_OUTPUT_MULTITHREADEDMEASUREMENT_H_ */ diff --git a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp index 6e0569105..a94d282e2 100644 --- a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp +++ b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp @@ -22,18 +22,17 @@ * Author: mschefer */ -#include #include +#include #include -#include "SEImplementation/Plugin/SourceIDs/SourceID.h" #include "SEImplementation/Measurement/MultithreadedMeasurement.h" +#include "SEImplementation/Plugin/SourceIDs/SourceID.h" using namespace SourceXtractor; static Elements::Logging logger = Elements::Logging::getLogger("Multithreading"); - MultithreadedMeasurement::~MultithreadedMeasurement() { if (m_output_thread->joinable()) { m_output_thread->join(); @@ -61,12 +60,10 @@ void MultithreadedMeasurement::synchronizeThreads() { std::unique_lock output_lock(m_output_queue_mutex); if (m_output_queue.empty()) { break; - } - else if (m_thread_pool->checkForException(false)) { + } else if (m_thread_pool->checkForException(false)) { logger.fatal() << "An exception was thrown from a worker thread"; m_thread_pool->checkForException(true); - } - else if (m_thread_pool->activeThreads() == 0) { + } else if (m_thread_pool->activeThreads() == 0) { throw Elements::Exception() << "No active threads and the queue is not empty! Please, report this as a bug"; } } @@ -82,31 +79,28 @@ void MultithreadedMeasurement::receiveSource(std::unique_ptr output_lock(m_output_queue_mutex); - m_output_queue.emplace_back(order_number, std::move(source_group)); + m_output_queue.emplace(order_number, std::move(source_group)); } m_new_output.notify_one(); }; - auto lambda_copyable = [lambda = std::make_shared(std::move(lambda))](){ - (*lambda)(); - }; + auto lambda_copyable = [lambda = std::make_shared(std::move(lambda))]() { (*lambda)(); }; m_thread_pool->submit(lambda_copyable); ++m_group_counter; } -void MultithreadedMeasurement::outputThreadStatic(MultithreadedMeasurement *measurement) { +void MultithreadedMeasurement::outputThreadStatic(MultithreadedMeasurement* measurement) { logger.debug() << "Starting output thread"; try { measurement->outputThreadLoop(); - } - catch (const Elements::Exception& e) { + } catch (const Elements::Exception& e) { logger.fatal() << "Output thread got an exception!"; logger.fatal() << e.what(); if (!measurement->m_abort_raised.exchange(true)) { @@ -118,6 +112,8 @@ void MultithreadedMeasurement::outputThreadStatic(MultithreadedMeasurement *meas } void MultithreadedMeasurement::outputThreadLoop() { + int next_id = 0; + while (m_thread_pool->activeThreads() > 0) { std::unique_lock output_lock(m_output_queue_mutex); @@ -126,19 +122,18 @@ void MultithreadedMeasurement::outputThreadLoop() { m_new_output.wait_for(output_lock, std::chrono::milliseconds(100)); } - // Process the output queue - while (!m_output_queue.empty()) { - auto next_source = std::move(m_output_queue.front()); - m_output_queue.pop_front(); - - // Flush events received before this source - while (!m_event_queue.empty() && m_event_queue.front().first <= next_source.first) { - sendProcessSignal(m_event_queue.front().second); - m_event_queue.pop_front(); - } + // Flush events + while (!m_event_queue.empty() && m_event_queue.front().first <= next_id) { + sendProcessSignal(m_event_queue.front().second); + m_event_queue.pop(); + } - // Flush source - sendSource(std::move(next_source.second)); + // Process the output queue + if (!m_output_queue.empty() && m_output_queue.top().first <= next_id) { + auto& next_source = const_cast&>(m_output_queue.top().second); + sendSource(std::move(next_source)); + m_output_queue.pop(); + ++next_id; } if (m_input_done && m_thread_pool->running() + m_thread_pool->queued() == 0 && m_output_queue.empty()) { @@ -149,5 +144,5 @@ void MultithreadedMeasurement::outputThreadLoop() { void MultithreadedMeasurement::receiveProcessSignal(const ProcessSourcesEvent& event) { std::unique_lock output_lock(m_output_queue_mutex); - m_event_queue.emplace_back(m_group_counter, event); + m_event_queue.emplace(m_group_counter, event); } From 35bdb17219e797ed1410a81b8706abee0a0e631e Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 13:37:06 +0200 Subject: [PATCH 22/35] Add cancel() method to measurement --- SEFramework/SEFramework/Pipeline/Measurement.h | 4 +++- .../Measurement/MultithreadedMeasurement.h | 7 ++++++- .../src/lib/Measurement/MultithreadedMeasurement.cpp | 6 +++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/SEFramework/SEFramework/Pipeline/Measurement.h b/SEFramework/SEFramework/Pipeline/Measurement.h index ece9264e3..60af09e58 100644 --- a/SEFramework/SEFramework/Pipeline/Measurement.h +++ b/SEFramework/SEFramework/Pipeline/Measurement.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -39,6 +40,7 @@ class Measurement : public PipelineReceiver, public Pipeli virtual void startThreads() = 0; virtual void stopThreads() = 0; virtual void synchronizeThreads() = 0; + virtual void cancel() = 0; }; } diff --git a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h index e7817fbb2..5e2f739ec 100644 --- a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h @@ -46,6 +46,7 @@ class MultithreadedMeasurement : public Measurement { , m_group_counter(0) , m_input_done(false) , m_abort_raised(false) + , m_cancel(false) , m_semaphore(max_queue_size) {} ~MultithreadedMeasurement() override; @@ -57,6 +58,10 @@ class MultithreadedMeasurement : public Measurement { void stopThreads() override; void synchronizeThreads() override; + void cancel() override { + m_cancel = true; + } + private: using QueuePair = std::pair>; // We want O(1) for the *lowest* value (received order) @@ -70,7 +75,7 @@ class MultithreadedMeasurement : public Measurement { std::unique_ptr m_output_thread; int m_group_counter; - std::atomic_bool m_input_done, m_abort_raised; + std::atomic_bool m_input_done, m_abort_raised, m_cancel; std::condition_variable m_new_output; OutputQueue m_output_queue; diff --git a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp index a94d282e2..12666bca1 100644 --- a/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp +++ b/SEImplementation/src/lib/Measurement/MultithreadedMeasurement.cpp @@ -80,6 +80,10 @@ void MultithreadedMeasurement::receiveSource(std::unique_ptractiveThreads() > 0) { + while (m_thread_pool->activeThreads() > 0 && !m_cancel) { std::unique_lock output_lock(m_output_queue_mutex); // Wait for something in the output queue From 46f1c95d452cf606e74a7cd730a9103e2ae23725 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 13:37:47 +0200 Subject: [PATCH 23/35] Fix handling of the timeout --- SEPythonModule/src/lib/FitsOutput.cpp | 1 + SEPythonModule/src/lib/Measurement.cpp | 1 + SEPythonModule/src/lib/NumpyOutput.cpp | 1 + SEPythonWrapper/scripts/SourceXtractorDemo.py | 4 +++- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/SEPythonModule/src/lib/FitsOutput.cpp b/SEPythonModule/src/lib/FitsOutput.cpp index 6e3eff6e7..61119d54b 100644 --- a/SEPythonModule/src/lib/FitsOutput.cpp +++ b/SEPythonModule/src/lib/FitsOutput.cpp @@ -76,6 +76,7 @@ void FitsOutput::get(std::chrono::microseconds timeout) { m_context->m_thread_pool->checkForException(true); timeout -= try_wait; if (timeout <= std::chrono::microseconds::zero()) { + Pyston::GILLocker gil; PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); py::throw_error_already_set(); } diff --git a/SEPythonModule/src/lib/Measurement.cpp b/SEPythonModule/src/lib/Measurement.cpp index e0272a33d..7acab927d 100644 --- a/SEPythonModule/src/lib/Measurement.cpp +++ b/SEPythonModule/src/lib/Measurement.cpp @@ -31,6 +31,7 @@ Measurement::Measurement(ContextPtr context) : m_context(std::move(context)) { Measurement::~Measurement() { try { + m_measurement->cancel(); m_measurement->stopThreads(); } catch (...) { // Must not throw from a destructor diff --git a/SEPythonModule/src/lib/NumpyOutput.cpp b/SEPythonModule/src/lib/NumpyOutput.cpp index 6a38f14ca..b193c7ef9 100644 --- a/SEPythonModule/src/lib/NumpyOutput.cpp +++ b/SEPythonModule/src/lib/NumpyOutput.cpp @@ -76,6 +76,7 @@ py::object NumpyOutput::getTable(std::chrono::microseconds timeout) { m_context->m_thread_pool->checkForException(true); timeout -= try_wait; if (timeout <= std::chrono::microseconds::zero()) { + Pyston::GILLocker gil; PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); py::throw_error_already_set(); } diff --git a/SEPythonWrapper/scripts/SourceXtractorDemo.py b/SEPythonWrapper/scripts/SourceXtractorDemo.py index 72bb60ad5..49dda932e 100644 --- a/SEPythonWrapper/scripts/SourceXtractorDemo.py +++ b/SEPythonWrapper/scripts/SourceXtractorDemo.py @@ -20,6 +20,7 @@ import os.path from argparse import ArgumentParser from configparser import ConfigParser +from datetime import timedelta from typing import Any, Dict import h5py @@ -108,6 +109,7 @@ def run_sourcextractor(config: Dict[str, Any], output_path: str, stamps: bool): Output = pipeline.FitsOutput config['output-catalog-filename'] = output_path + timeout = config.pop('timeout', timedelta(days=365)) snr_filter = SNRFilter(float(config.pop('snr', 5))) with pipeline.Context(config): stages = [pipeline.Segmentation(), pipeline.Partition(), snr_filter, pipeline.Grouping(), pipeline.Deblending()] @@ -115,7 +117,7 @@ def run_sourcextractor(config: Dict[str, Any], output_path: str, stamps: bool): stages.append(StoreStamps(h5)) stages.extend([pipeline.Measurement(), Output()]) pipe = pipeline.Pipeline(stages) - result = pipe().get() + result = pipe().get(timeout=timeout) print(f'Dropped {len(snr_filter.dropped)} sources') if h5 is not None: From 59ad317c579fb3f4862755fe73a726dee5647a38 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 13 Oct 2022 14:06:38 +0200 Subject: [PATCH 24/35] Forgot to override cancel in DummyMeasurement --- .../SEImplementation/Measurement/DummyMeasurement.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h b/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h index 84cd40113..f35083d25 100644 --- a/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/DummyMeasurement.h @@ -1,4 +1,5 @@ -/** Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -42,6 +43,7 @@ class DummyMeasurement : public Measurement { void startThreads() override {}; void stopThreads() override {}; void synchronizeThreads() override {}; + void cancel() override {}; }; } From 0485f57a18ce83e63e40c1d4d9ff4d2d0ead02fc Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 14 Oct 2022 10:52:45 +0200 Subject: [PATCH 25/35] Add Context::check_exception --- SEPythonModule/SEPythonModule/Context.h | 3 +++ SEPythonModule/src/lib/Context.cpp | 4 ++++ SEPythonModule/src/lib/SePyMain.cpp | 1 + 3 files changed, 8 insertions(+) diff --git a/SEPythonModule/SEPythonModule/Context.h b/SEPythonModule/SEPythonModule/Context.h index 548195c15..856e92678 100644 --- a/SEPythonModule/SEPythonModule/Context.h +++ b/SEPythonModule/SEPythonModule/Context.h @@ -76,6 +76,9 @@ class Context { std::function m_source_to_row; std::shared_ptr m_thread_pool; + /// Rethrow an exception if a worker thread failed + void check_exception() const; + static void enter(const std::shared_ptr&); static void exit(const std::shared_ptr&, const boost::python::object& exc_type, const boost::python::object& exc_value, const boost::python::object& traceback); diff --git a/SEPythonModule/src/lib/Context.cpp b/SEPythonModule/src/lib/Context.cpp index 55b672812..b9855da1d 100644 --- a/SEPythonModule/src/lib/Context.cpp +++ b/SEPythonModule/src/lib/Context.cpp @@ -124,6 +124,10 @@ py::dict Context::get_properties() const { return properties; } +void Context::check_exception() const { + m_thread_pool->checkForException(true); +} + void Context::enter(const std::shared_ptr& context) { s_context = context; } diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index 7defbfde1..f6e9c7e01 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -116,6 +116,7 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { py::class_( "Context", py::init((py::arg("config"), py::arg("measurement_config") = py::object()))) .def("get_properties", &Context::get_properties) + .def("check_exception", &Context::check_exception) .def("__enter__", &Context::enter) .def("__exit__", &Context::exit); py::register_ptr_to_python>(); From c8d0f01f7b51656a863acb3b96e7236f5b84a677 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 14 Oct 2022 12:19:30 +0200 Subject: [PATCH 26/35] Fix timedelta conversion --- SEPythonModule/src/lib/SePyMain.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index f6e9c7e01..ac7d12f48 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -91,7 +91,7 @@ struct std_duration_from_timedelta { static void construct(PyObject* obj_ptr, py::converter::rvalue_from_python_stage1_data* data) { auto timedelta = reinterpret_cast(obj_ptr); - auto days = std::chrono::hours(PyDateTime_DELTA_GET_DAYS(timedelta) * 24); + auto days = std::chrono::hours(PyDateTime_DELTA_GET_DAYS(timedelta) * 24l); auto seconds = std::chrono::seconds(PyDateTime_DELTA_GET_SECONDS(timedelta)); auto microseconds = std::chrono::microseconds(PyDateTime_DELTA_GET_MICROSECONDS(timedelta)); auto duration = days + seconds + microseconds; From dc3c234a56fcf4365f05c574f3e1ec2e029034e9 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 14 Oct 2022 13:58:05 +0200 Subject: [PATCH 27/35] Make PixelCoordinateList moveable --- .../Property/PixelCoordinateList.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/SEImplementation/SEImplementation/Property/PixelCoordinateList.h b/SEImplementation/SEImplementation/Property/PixelCoordinateList.h index 92e889fa4..e70f374dd 100644 --- a/SEImplementation/SEImplementation/Property/PixelCoordinateList.h +++ b/SEImplementation/SEImplementation/Property/PixelCoordinateList.h @@ -1,4 +1,5 @@ -/** Copyright © 2019 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +/** + * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License as published by the Free @@ -14,7 +15,7 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PixelCoordinateList.h * @author nikoapos */ @@ -29,15 +30,18 @@ namespace SourceXtractor { class PixelCoordinateList : public Property { - + public: - + explicit PixelCoordinateList(std::vector coordinate_list) : m_coordinate_list(std::move(coordinate_list)) { } + PixelCoordinateList(const PixelCoordinateList&) = default; + PixelCoordinateList(PixelCoordinateList&&) = default; + virtual ~PixelCoordinateList() = default; - + const std::vector& getCoordinateList() const { return m_coordinate_list; } @@ -45,11 +49,11 @@ class PixelCoordinateList : public Property { bool contains(const PixelCoordinate& coord) const { return std::find(m_coordinate_list.begin(), m_coordinate_list.end(), coord) != m_coordinate_list.end(); } - + private: std::vector m_coordinate_list; - + }; /* End of PixelCoordinateList class */ } /* namespace SourceXtractor */ From bc9d776153f4e824ba683852ec8d90e71d59814b Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 14 Oct 2022 12:37:06 +0200 Subject: [PATCH 28/35] Custom segmentation --- .../SEPythonModule/PipelineReceiver.h | 13 +- .../SEPythonModule/SourceInterface.h | 5 + SEPythonModule/src/lib/Deblending.cpp | 5 +- SEPythonModule/src/lib/FitsOutput.cpp | 5 +- SEPythonModule/src/lib/Grouping.cpp | 5 +- SEPythonModule/src/lib/Measurement.cpp | 5 +- SEPythonModule/src/lib/NumpyOutput.cpp | 2 +- SEPythonModule/src/lib/Partition.cpp | 5 +- SEPythonModule/src/lib/PipelineReceiver.cpp | 43 +++++ SEPythonModule/src/lib/SePyMain.cpp | 11 +- SEPythonModule/src/lib/Segmentation.cpp | 19 +-- SEPythonModule/src/lib/SourceInterface.cpp | 36 +++++ .../scripts/CustomSegmentationDemo.py | 152 ++++++++++++++++++ 13 files changed, 268 insertions(+), 38 deletions(-) create mode 100644 SEPythonModule/src/lib/PipelineReceiver.cpp create mode 100644 SEPythonWrapper/scripts/CustomSegmentationDemo.py diff --git a/SEPythonModule/SEPythonModule/PipelineReceiver.h b/SEPythonModule/SEPythonModule/PipelineReceiver.h index 566766df7..eab2d19e2 100644 --- a/SEPythonModule/SEPythonModule/PipelineReceiver.h +++ b/SEPythonModule/SEPythonModule/PipelineReceiver.h @@ -31,16 +31,13 @@ namespace SourceXPy { using SourceReceiverIfce = SourceXtractor::PipelineReceiver; using GroupReceiverIfce = SourceXtractor::PipelineReceiver; -class ProcessSourcesEvent { -public: - SourceXtractor::ProcessSourcesEvent m_event; - - std::string repr() const; -}; +std::string ProcessSourcesEventRepr(const SourceXtractor::ProcessSourcesEvent&); class AllFramesDone : public SourceXtractor::SelectionCriteria { public: bool mustBeProcessed(const SourceXtractor::SourceInterface& source) const override; + + static SourceXtractor::ProcessSourcesEvent create(); }; class PipelineSourceReceiver : public SourceReceiverIfce { @@ -62,7 +59,7 @@ class PipelineSourceReceiver : public SourceReceiverIfce { void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { Pyston::GILLocker gil; try { - m_callback(ProcessSourcesEvent{event}); + m_callback(event); } catch (const boost::python::error_already_set&) { throw Pyston::Exception(); } @@ -92,7 +89,7 @@ class PipelineGroupReceiver : public GroupReceiverIfce { void receiveProcessSignal(const SourceXtractor::ProcessSourcesEvent& event) override { Pyston::GILLocker gil; try { - m_callback(ProcessSourcesEvent{event}); + m_callback(event); } catch (const boost::python::error_already_set&) { throw Pyston::Exception(); } diff --git a/SEPythonModule/SEPythonModule/SourceInterface.h b/SEPythonModule/SEPythonModule/SourceInterface.h index 78f85be53..d38961d4c 100644 --- a/SEPythonModule/SEPythonModule/SourceInterface.h +++ b/SEPythonModule/SEPythonModule/SourceInterface.h @@ -21,7 +21,9 @@ #include "SEFramework/Source/SourceGroupInterface.h" #include "SEFramework/Source/SourceInterface.h" +#include "SEImplementation/Property/PixelCoordinateList.h" #include "SEPythonModule/Context.h" +#include #include #include #include @@ -54,6 +56,9 @@ struct OwnedSource : public AttachedSource { std::string repr() const; std::unique_ptr m_owned_source; + + static std::shared_ptr create(const std::shared_ptr& context, int detection_frame_idx, + int detection_id, const boost::python::tuple& pixels); }; struct EntangledSource : public AttachedSource { diff --git a/SEPythonModule/src/lib/Deblending.cpp b/SEPythonModule/src/lib/Deblending.cpp index 28d077f00..43d93d70b 100644 --- a/SEPythonModule/src/lib/Deblending.cpp +++ b/SEPythonModule/src/lib/Deblending.cpp @@ -23,6 +23,7 @@ #include namespace py = boost::python; +namespace se = SourceXtractor; namespace SourceXPy { @@ -56,9 +57,9 @@ void Deblending::call(const py::object& obj) const { m_deblending->receiveSource(std::move(cloned_group_ptr)); return; } - py::extract event_wrapper(obj); + py::extract event_wrapper(obj); if (event_wrapper.check()) { - const auto& event = event_wrapper().m_event; + const auto& event = event_wrapper(); Pyston::SaveThread save_thread; m_deblending->receiveProcessSignal(event); return; diff --git a/SEPythonModule/src/lib/FitsOutput.cpp b/SEPythonModule/src/lib/FitsOutput.cpp index 61119d54b..fb09b3083 100644 --- a/SEPythonModule/src/lib/FitsOutput.cpp +++ b/SEPythonModule/src/lib/FitsOutput.cpp @@ -23,6 +23,7 @@ #include namespace py = boost::python; +namespace se = SourceXtractor; namespace SourceXPy { @@ -58,9 +59,9 @@ void FitsOutput::call(const boost::python::object& obj) { m_output->receiveSource(std::move(cloned_group_ptr)); return; } - py::extract event_wrapper(obj); + py::extract event_wrapper(obj); if (event_wrapper.check()) { - const auto& event = event_wrapper().m_event; + const auto& event = event_wrapper(); Pyston::SaveThread save_thread; m_output->receiveProcessSignal(event); return; diff --git a/SEPythonModule/src/lib/Grouping.cpp b/SEPythonModule/src/lib/Grouping.cpp index 1178afeca..2e94e1851 100644 --- a/SEPythonModule/src/lib/Grouping.cpp +++ b/SEPythonModule/src/lib/Grouping.cpp @@ -22,6 +22,7 @@ #include namespace py = boost::python; +namespace se = SourceXtractor; namespace SourceXPy { @@ -58,9 +59,9 @@ void Grouping::call(const py::object& obj) const { m_grouping->receiveSource(source_ptr->clone()); return; } - py::extract event_wrapper(obj); + py::extract event_wrapper(obj); if (event_wrapper.check()) { - const auto& event = event_wrapper().m_event; + const auto& event = event_wrapper(); Pyston::SaveThread save_thread; m_grouping->receiveProcessSignal(event); return; diff --git a/SEPythonModule/src/lib/Measurement.cpp b/SEPythonModule/src/lib/Measurement.cpp index 7acab927d..ffb95999c 100644 --- a/SEPythonModule/src/lib/Measurement.cpp +++ b/SEPythonModule/src/lib/Measurement.cpp @@ -22,6 +22,7 @@ #include namespace py = boost::python; +namespace se = SourceXtractor; namespace SourceXPy { @@ -70,9 +71,9 @@ void Measurement::call(const py::object& obj) { m_measurement->receiveSource(std::move(cloned_group_ptr)); return; } - py::extract event_wrapper(obj); + py::extract event_wrapper(obj); if (event_wrapper.check()) { - const auto& event = event_wrapper().m_event; + const auto& event = event_wrapper(); Pyston::SaveThread save_thread; m_measurement->receiveProcessSignal(event); return; diff --git a/SEPythonModule/src/lib/NumpyOutput.cpp b/SEPythonModule/src/lib/NumpyOutput.cpp index b193c7ef9..94daef63f 100644 --- a/SEPythonModule/src/lib/NumpyOutput.cpp +++ b/SEPythonModule/src/lib/NumpyOutput.cpp @@ -61,7 +61,7 @@ void NumpyOutput::call(const py::object& obj) { boost::transform(*group_ptr, std::back_inserter(m_rows), m_source_to_row); return; } - py::extract event_wrapper(obj); + py::extract event_wrapper(obj); if (!event_wrapper.check()) { PyErr_SetString(PyExc_TypeError, "NumpyOutput: Unexpected python object received"); py::throw_error_already_set(); diff --git a/SEPythonModule/src/lib/Partition.cpp b/SEPythonModule/src/lib/Partition.cpp index 2341ea82d..a19859e7b 100644 --- a/SEPythonModule/src/lib/Partition.cpp +++ b/SEPythonModule/src/lib/Partition.cpp @@ -22,6 +22,7 @@ #include namespace py = boost::python; +namespace se = SourceXtractor; namespace SourceXPy { @@ -58,9 +59,9 @@ void Partition::call(const py::object& obj) const { m_partition->receiveSource(source_ptr->clone()); return; } - py::extract event_wrapper(obj); + py::extract event_wrapper(obj); if (event_wrapper.check()) { - const auto& event = event_wrapper().m_event; + const auto& event = event_wrapper(); Pyston::SaveThread save_thread; m_partition->receiveProcessSignal(event); return; diff --git a/SEPythonModule/src/lib/PipelineReceiver.cpp b/SEPythonModule/src/lib/PipelineReceiver.cpp new file mode 100644 index 000000000..64d5823f1 --- /dev/null +++ b/SEPythonModule/src/lib/PipelineReceiver.cpp @@ -0,0 +1,43 @@ +/** + * Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université + * + * This library is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License as published by the Free + * Software Foundation; either version 3.0 of the License, or (at your option) + * any later version. + * + * This library is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + * details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this library; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "SEPythonModule/PipelineReceiver.h" + +namespace se = SourceXtractor; + +namespace SourceXPy { + +std::string ProcessSourcesEventRepr(const se::ProcessSourcesEvent& event) { + auto& ptr = event.m_selection_criteria; + if (std::dynamic_pointer_cast(ptr)) { + return "SelectAllCriteria"; + } else if (std::dynamic_pointer_cast(ptr)) { + return "AllFramesDone"; + } + return "LineSelectionCriteria"; +} + +bool AllFramesDone::mustBeProcessed(const se::SourceInterface&) const { + return true; +} + +se::ProcessSourcesEvent AllFramesDone::create() { + return se::ProcessSourcesEvent{std::make_shared()}; +} + +} // namespace SourceXPy diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index ac7d12f48..180c64497 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -37,6 +37,7 @@ using namespace SourceXPy; namespace py = boost::python; namespace np = boost::python::numpy; +namespace se = SourceXtractor; namespace { @@ -129,7 +130,9 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { py::class_("Source", py::no_init) .def("__getattr__", &AttachedSource::attribute) - .def("detach", &AttachedSource::detach); + .def("detach", &AttachedSource::detach) + .def("create", &OwnedSource::create, py::args("context", "detection_frame", "detection_id", "pixel_coordinates")) + .staticmethod("create"); py::register_ptr_to_python>(); py::class_, boost::noncopyable>("OwnedSource", py::no_init) @@ -150,7 +153,7 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { .def("__iter__", &SourceGroup::iter); py::register_ptr_to_python>(); - py::class_("ProcessSourcesEvent", py::no_init).def("__repr__", &ProcessSourcesEvent::repr); + py::class_("ProcessSourcesEvent", py::no_init).def("__repr__", &ProcessSourcesEventRepr); py::class_("SourceReceiver", py::no_init); py::class_("GroupReceiverIfce", py::no_init); @@ -196,6 +199,10 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { .def("__call__", &FitsOutput::call) .def("get", &FitsOutput::get, (py::arg("timeout") = std::chrono::microseconds::max())); + // For custom segmentation + py::def("AllFramesDone", &AllFramesDone::create); + + // Import pyston into the interpreter so it is importable without tweaking PYTHONPATH #if PY_MAJOR_VERSION >= 3 PyObject* pyston = PyInit_pyston(); #else diff --git a/SEPythonModule/src/lib/Segmentation.cpp b/SEPythonModule/src/lib/Segmentation.cpp index 788c7878e..58ec57802 100644 --- a/SEPythonModule/src/lib/Segmentation.cpp +++ b/SEPythonModule/src/lib/Segmentation.cpp @@ -28,11 +28,10 @@ namespace SourceXPy { namespace py = boost::python; -namespace se = SourceXtractor; using SourceXtractor::DetectionFrameConfig; using SourceXtractor::DetectionImageFrame; -using SourceXtractor::SelectAllCriteria; +using SourceXtractor::ProcessSourcesEvent; Segmentation::Segmentation(ContextPtr context) : m_context(std::move(context)) { m_segmentation = m_context->m_segmentation_factory->createSegmentation(); @@ -58,21 +57,7 @@ void Segmentation::call() const { const auto& frames = detection_config.getDetectionFrames(); boost::for_each(frames, [this](const std::shared_ptr& frame) { m_segmentation->processFrame(frame); }); - m_next_stage->receiveProcessSignal(se::ProcessSourcesEvent(std::make_shared())); -} - -std::string ProcessSourcesEvent::repr() const { - auto& ptr = m_event.m_selection_criteria; - if (std::dynamic_pointer_cast(ptr)) { - return "SelectAllCriteria"; - } else if (std::dynamic_pointer_cast(ptr)) { - return "AllFramesDone"; - } - return "LineSelectionCriteria"; -} - -bool AllFramesDone::mustBeProcessed(const SourceXtractor::SourceInterface&) const { - return true; + m_next_stage->receiveProcessSignal(ProcessSourcesEvent(std::make_shared())); } } // namespace SourceXPy diff --git a/SEPythonModule/src/lib/SourceInterface.cpp b/SEPythonModule/src/lib/SourceInterface.cpp index 3ec051af4..075b9f0ca 100644 --- a/SEPythonModule/src/lib/SourceInterface.cpp +++ b/SEPythonModule/src/lib/SourceInterface.cpp @@ -18,9 +18,11 @@ #include "SEPythonModule/SourceInterface.h" #include "SEFramework/Output/OutputRegistry.h" +#include "SEFramework/Property/DetectionFrame.h" #include "SEFramework/Source/SourceFactory.h" #include "SEFramework/Source/SourceGroupWithOnDemandPropertiesFactory.h" #include "SEFramework/Source/SourceInterface.h" +#include "SEImplementation/Configuration/DetectionFrameConfig.h" #include "SEImplementation/Plugin/GroupInfo/GroupInfo.h" #include "SEImplementation/Property/SourceId.h" #include @@ -34,7 +36,10 @@ namespace SourceXPy { namespace py = boost::python; namespace np = boost::python::numpy; using Euclid::NdArray::NdArray; +using SourceXtractor::DetectionFrame; +using SourceXtractor::DetectionFrameConfig; using SourceXtractor::GroupInfo; +using SourceXtractor::PixelCoordinateList; using SourceXtractor::PropertyId; using SourceXtractor::SourceId; @@ -191,4 +196,35 @@ std::shared_ptr SourceGroup::Iterator::next() { return m_holder; } +namespace { + +PixelCoordinateList PixelCoordinateFromTuple(const boost::python::tuple& tuple) { + const np::ndarray xs = py::extract(tuple[0]); + const np::ndarray ys = py::extract(tuple[1]); + std::vector coordinates(py::len(xs)); + + for (std::size_t i = 0; i < coordinates.size(); ++i) { + coordinates[i].m_x = py::extract(xs[i]); + coordinates[i].m_y = py::extract(ys[i]); + } + + return PixelCoordinateList{std::move(coordinates)}; +} + +} // namespace + +std::shared_ptr OwnedSource::create(const std::shared_ptr& context, int detection_frame_idx, + int detection_id, const boost::python::tuple& pixels) { + const auto& detection_frames = + context->m_config_manager->getConfiguration().getDetectionFrames(); + const auto& detection_frame = detection_frames.at(detection_frame_idx); + + auto source_ptr = context->m_source_factory->createSource(); + source_ptr->setProperty(detection_frame); + source_ptr->setProperty(detection_id); + source_ptr->setProperty(PixelCoordinateFromTuple(pixels)); + + return std::make_shared(context, std::move(source_ptr)); +} + } // namespace SourceXPy diff --git a/SEPythonWrapper/scripts/CustomSegmentationDemo.py b/SEPythonWrapper/scripts/CustomSegmentationDemo.py new file mode 100644 index 000000000..1351c4c1a --- /dev/null +++ b/SEPythonWrapper/scripts/CustomSegmentationDemo.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import itertools +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict, Tuple + +import numpy as np +from astropy.io import fits +from scipy.ndimage import find_objects, label +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + + +class CustomSegmentation: + """ + Naive segmentation algorithm based on a value / error threshold and scipy.ndimage.label + This is just an example! There is no background subtraction, nor filtering. + Everything is done in memory, so big images will fill the system memory. + + :param context: + sourcextractor++ context + :param detection: + Detection image + :param weight: + Weight image + :param weight_type: + Weight type. Only 'weight', 'variance' and 'rms' are supported. + :param snr: + Threshold for the value / error check + """ + + @staticmethod + def __setup_error(weight_img: np.ndarray, weight_type: str): + """ + Transform the weight image into an error image (sqrt(variance)) + """ + if weight_type == 'weight': + return np.sqrt(np.reciprocal(weight_img)) + elif weight_type == 'variance': + return np.sqrt(weight_img) + elif weight_type == 'rms': + return weight_type + raise ValueError(f'Unsupported weight type {weight_type}') + + def __init__(self, context: pipeline.Context, detection: str, weight: str, weight_type: str, snr: float): + self.__context = context + self.__count = 0 + self.__detection = fits.open(detection)[0].data + self.__std = self.__setup_error(fits.open(weight)[0].data, weight_type) + self.__snr = snr + self.__next = None + + @property + def count(self): + """ + :return: The number of detected sources + """ + return self.__count + + def set_next_stage(self, next_stage): + """ + Set the following stage, which must receive sources + """ + self.__next = next_stage + + @staticmethod + def __get_pixel_coordinates(labeled: np.ndarray, obj: Tuple[slice, slice], idx: int): + """ + Given the labeled image, the object bounding box and its detection ID, return the list + of detected pixels as a tuple (xs, ys) + """ + cutout = labeled[obj] + # numpy axes are 0=Y, 1=X!! + yoff, xoff = np.nonzero(cutout == idx) + return (xoff + obj[1].start).astype(np.int32), (yoff + obj[0].start).astype(np.int32) + + def __call__(self): + """ + Apply the threshold, label the image, and find the objects + """ + # Mask out pixels below the threshold + mask = (self.__detection / self.__std) < self.__snr + self.__detection[mask] = 0. + # Use scipy label + labeled = label(self.__detection > 0)[0] + # Use scipy find_object to find the bounding boxes + detected_objs = find_objects(labeled) + for idx, obj in enumerate(detected_objs, start=1): + self.__count += 1 + pixels = self.__get_pixel_coordinates(labeled, obj, idx) + # We only support a single detection frame in this example + source = pipeline.Source.create(self.__context, detection_frame=0, detection_id=idx, + pixel_coordinates=pixels) + self.__next(source) + self.__next(pipeline.AllFramesDone()) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str): + """ + Setup the sourcextractor++ pipeline with a custom segmentation stage + """ + config['output-catalog-filename'] = output_path + snr = float(config.pop('snr', 5.)) + context = pipeline.Context(config) + # Generate out custom segmentation + segmentation = CustomSegmentation(context, config['detection-image'], config['weight-image'], config['weight-type'], + snr) + with context: + stages = [segmentation, pipeline.Partition(), pipeline.Grouping(), pipeline.Deblending(), + pipeline.Measurement(), pipeline.FitsOutput()] + pipe = pipeline.Pipeline(stages) + pipe().get() + print(f'Done! Found {segmentation.count} sources') + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +if __name__ == '__main__': + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='FITS', default='output.fits', help='Output file') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file) From 2f0dd747ca676b6b3b004df4d45f668ffba04508 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 14 Oct 2022 14:23:33 +0200 Subject: [PATCH 29/35] Add missing comments to Source python wrappers --- .../SEPythonModule/SourceInterface.h | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/SEPythonModule/SEPythonModule/SourceInterface.h b/SEPythonModule/SEPythonModule/SourceInterface.h index d38961d4c..aac15ce79 100644 --- a/SEPythonModule/SEPythonModule/SourceInterface.h +++ b/SEPythonModule/SEPythonModule/SourceInterface.h @@ -30,6 +30,10 @@ namespace SourceXPy { +/** + * A DetachedSource is "outside" sourcextractor++'s pipeline, so it does not keep any + * reference to internal properties such as the detection frame + */ struct DetachedSource { boost::python::dict m_attributes; @@ -38,6 +42,9 @@ struct DetachedSource { boost::python::object attribute(const std::string& key) const; }; +/** + * An AttachedSource is bound to sourcextractor++'s pipeline. It can not be serialized. + */ struct AttachedSource { ContextPtr m_context; SourceXtractor::SourceInterface* m_source_ptr = nullptr; @@ -49,6 +56,13 @@ struct AttachedSource { DetachedSource detach() const; }; +/** + * An OwnedSource is fully owned by the pipeline stage that receives it. + * It is still attached to the pipeline, but it is safe to keep a reference to it from Python + * @warning This is only true since the pipelines clone the sources that come from Python, which is an inefficiency. + * If acceptable, m_owned_source could be moved away, m_source_ptr set to nullptr, and let any later call + * catch this nullptr situation if the caller kept a reference without explicitly cloning. + */ struct OwnedSource : public AttachedSource { OwnedSource(ContextPtr context, std::unique_ptr source) : AttachedSource(std::move(context), source.get()), m_owned_source(std::move(source)) {} @@ -61,12 +75,25 @@ struct OwnedSource : public AttachedSource { int detection_id, const boost::python::tuple& pixels); }; +/** + * An EntangledSource lifetime is bound to its containing SourceGroup. When iterating a SourceGroup, + * only the current EntangledSource is safe to use. i.e. + * + * sources = [] + * for source in group: + * print(source) # Safe + * sources.append(source) # Unsafe + * print(sources) # Unsafe + */ struct EntangledSource : public AttachedSource { explicit EntangledSource(ContextPtr context) : AttachedSource(std::move(context), nullptr) {} std::string repr() const; }; +/** + * A SourceGroup is always owned by the called pipeline stage + */ struct SourceGroup { struct Iterator { SourceXtractor::SourceGroupInterface::const_iterator m_i; From c95850a7fe86eab39f559c9cd40c953db20465aa Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Fri, 14 Oct 2022 15:23:45 +0200 Subject: [PATCH 30/35] Fix build in centos7 --- .../Measurement/MultithreadedMeasurement.h | 2 +- SEPythonModule/src/lib/FitsOutput.cpp | 4 ++++ SEPythonModule/src/lib/NumpyOutput.cpp | 4 ++++ SEPythonModule/src/lib/SePyMain.cpp | 8 ++++---- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h index 5e2f739ec..50f34d860 100644 --- a/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h +++ b/SEImplementation/SEImplementation/Measurement/MultithreadedMeasurement.h @@ -65,7 +65,7 @@ class MultithreadedMeasurement : public Measurement { private: using QueuePair = std::pair>; // We want O(1) for the *lowest* value (received order) - using OutputQueue = std::priority_queue, std::greater<>>; + using OutputQueue = std::priority_queue, std::greater>; static void outputThreadStatic(MultithreadedMeasurement* measurement); void outputThreadLoop(); diff --git a/SEPythonModule/src/lib/FitsOutput.cpp b/SEPythonModule/src/lib/FitsOutput.cpp index fb09b3083..6640d28c9 100644 --- a/SEPythonModule/src/lib/FitsOutput.cpp +++ b/SEPythonModule/src/lib/FitsOutput.cpp @@ -78,7 +78,11 @@ void FitsOutput::get(std::chrono::microseconds timeout) { timeout -= try_wait; if (timeout <= std::chrono::microseconds::zero()) { Pyston::GILLocker gil; +#if PY_VERSION_HEX >= 0x03030000 PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); +#else + PyErr_SetString(PyExc_RuntimeError, "sourcextractor timed-out"); +#endif py::throw_error_already_set(); } } diff --git a/SEPythonModule/src/lib/NumpyOutput.cpp b/SEPythonModule/src/lib/NumpyOutput.cpp index 94daef63f..e56370e28 100644 --- a/SEPythonModule/src/lib/NumpyOutput.cpp +++ b/SEPythonModule/src/lib/NumpyOutput.cpp @@ -77,7 +77,11 @@ py::object NumpyOutput::getTable(std::chrono::microseconds timeout) { timeout -= try_wait; if (timeout <= std::chrono::microseconds::zero()) { Pyston::GILLocker gil; +#if PY_VERSION_HEX >= 0x03030000 PyErr_SetString(PyExc_TimeoutError, "sourcextractor timed-out"); +#else + PyErr_SetString(PyExc_RuntimeError, "sourcextractor timed-out"); +#endif py::throw_error_already_set(); } } diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index 180c64497..a81fc92d0 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -92,10 +92,10 @@ struct std_duration_from_timedelta { static void construct(PyObject* obj_ptr, py::converter::rvalue_from_python_stage1_data* data) { auto timedelta = reinterpret_cast(obj_ptr); - auto days = std::chrono::hours(PyDateTime_DELTA_GET_DAYS(timedelta) * 24l); - auto seconds = std::chrono::seconds(PyDateTime_DELTA_GET_SECONDS(timedelta)); - auto microseconds = std::chrono::microseconds(PyDateTime_DELTA_GET_MICROSECONDS(timedelta)); - auto duration = days + seconds + microseconds; + auto hours = std::chrono::hours(timedelta->days * 24l); + auto seconds = std::chrono::seconds(timedelta->seconds); + auto microseconds = std::chrono::microseconds(timedelta->microseconds); + auto duration = hours + seconds + microseconds; auto storage = (reinterpret_cast*>(data))->storage.bytes; From a308bc728c9c1c357a4e19a542b1de384f6f358d Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 17 Oct 2022 12:16:13 +0200 Subject: [PATCH 31/35] Add support for custom grouping --- .../SEPythonModule/PipelineReceiver.h | 2 + .../SEPythonModule/SourceInterface.h | 3 + SEPythonModule/src/lib/PipelineReceiver.cpp | 4 + SEPythonModule/src/lib/SePyMain.cpp | 8 +- SEPythonModule/src/lib/SourceInterface.cpp | 19 +- SEPythonWrapper/conf/SourceXtractorDemo.conf | 2 +- SEPythonWrapper/scripts/CustomGroupingDemo.py | 174 ++++++++++++++++++ 7 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 SEPythonWrapper/scripts/CustomGroupingDemo.py diff --git a/SEPythonModule/SEPythonModule/PipelineReceiver.h b/SEPythonModule/SEPythonModule/PipelineReceiver.h index eab2d19e2..8bc9dd27a 100644 --- a/SEPythonModule/SEPythonModule/PipelineReceiver.h +++ b/SEPythonModule/SEPythonModule/PipelineReceiver.h @@ -33,6 +33,8 @@ using GroupReceiverIfce = SourceXtractor::PipelineReceiver +#include #include #include #include @@ -111,6 +112,8 @@ struct SourceGroup { size_t size() const; boost::python::object attribute(const std::string& key) const; Iterator iter() const; + + static std::shared_ptr create(const std::shared_ptr& context, boost::python::list& sources); }; } // namespace SourceXPy diff --git a/SEPythonModule/src/lib/PipelineReceiver.cpp b/SEPythonModule/src/lib/PipelineReceiver.cpp index 64d5823f1..27c2e55e5 100644 --- a/SEPythonModule/src/lib/PipelineReceiver.cpp +++ b/SEPythonModule/src/lib/PipelineReceiver.cpp @@ -32,6 +32,10 @@ std::string ProcessSourcesEventRepr(const se::ProcessSourcesEvent& event) { return "LineSelectionCriteria"; } +bool ProcessSourcesEventMustProcess(const SourceXtractor::ProcessSourcesEvent& event, const AttachedSource& source) { + return event.m_selection_criteria->mustBeProcessed(*source.m_source_ptr); +} + bool AllFramesDone::mustBeProcessed(const se::SourceInterface&) const { return true; } diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index a81fc92d0..7d6dafd77 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -150,10 +150,14 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { .def("__repr__", &SourceGroup::repr) .def("__getattr__", &SourceGroup::attribute) .def("__len__", &SourceGroup::size) - .def("__iter__", &SourceGroup::iter); + .def("__iter__", &SourceGroup::iter) + .def("create", &SourceGroup::create) + .staticmethod("create"); py::register_ptr_to_python>(); - py::class_("ProcessSourcesEvent", py::no_init).def("__repr__", &ProcessSourcesEventRepr); + py::class_("ProcessSourcesEvent", py::no_init) + .def("__repr__", &ProcessSourcesEventRepr) + .def("must_process", &ProcessSourcesEventMustProcess); py::class_("SourceReceiver", py::no_init); py::class_("GroupReceiverIfce", py::no_init); diff --git a/SEPythonModule/src/lib/SourceInterface.cpp b/SEPythonModule/src/lib/SourceInterface.cpp index 075b9f0ca..5e7eba404 100644 --- a/SEPythonModule/src/lib/SourceInterface.cpp +++ b/SEPythonModule/src/lib/SourceInterface.cpp @@ -114,9 +114,7 @@ py::object AttachedSource::attribute(const std::string& key) const { PropertyId property_id(property_type_index, 0); m_source_ptr->getProperty(property_id); } catch (const std::out_of_range&) { - std::stringstream err_str("Source has no attribute "); - err_str << key; - PyErr_SetString(PyExc_AttributeError, err_str.str().c_str()); + PyErr_SetString(PyExc_AttributeError, key.c_str()); py::throw_error_already_set(); } // Convert @@ -227,4 +225,19 @@ std::shared_ptr OwnedSource::create(const std::shared_ptr& return std::make_shared(context, std::move(source_ptr)); } +std::shared_ptr SourceGroup::create(const std::shared_ptr& context, + boost::python::list& sources) { + auto group_ptr = context->m_group_factory->createSourceGroup(); + + auto n_sources = len(sources); + for (ssize_t i = 0; i < n_sources; ++i) { + const auto& source = py::extract>(sources[i]); + // TODO: Do not clone? + group_ptr->addSource(source()->m_owned_source->clone()); + } + + sources.attr("clear")(); + return std::make_shared(SourceGroup{std::move(group_ptr), context}); +} + } // namespace SourceXPy diff --git a/SEPythonWrapper/conf/SourceXtractorDemo.conf b/SEPythonWrapper/conf/SourceXtractorDemo.conf index 486e0c264..42bf1fff2 100644 --- a/SEPythonWrapper/conf/SourceXtractorDemo.conf +++ b/SEPythonWrapper/conf/SourceXtractorDemo.conf @@ -24,7 +24,7 @@ partition-minimum-contrast = 0.005 psf-fwhm = 3 psf-pixel-sampling = 1 weight-use-symmetry = 1 -output-properties = SourceIDs,PixelCentroid,WorldCentroid,IsophotalFlux,SourceFlags +output-properties = SourceIDs,GroupInfo,PixelCentroid,WorldCentroid,ShapeParameters,IsophotalFlux,SourceFlags detection-image = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/img/sim12.fits.gz weight-image = /home/aalvarez/Work/Projects/SourceXtractor-litmus/tests/../data/sim12/img/sim12.weight.fits.gz weight-type = weight diff --git a/SEPythonWrapper/scripts/CustomGroupingDemo.py b/SEPythonWrapper/scripts/CustomGroupingDemo.py new file mode 100644 index 000000000..03cd0b084 --- /dev/null +++ b/SEPythonWrapper/scripts/CustomGroupingDemo.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +import itertools +import logging +import sys +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict, List, Union + +import numpy as np +from sklearn.cluster import AgglomerativeClustering +from sklearn.neighbors import radius_neighbors_graph +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + +logger = logging.getLogger(__name__) + + +class CustomGrouping: + """ + Grouping based on AgglomerativeClustering. This is just an example of how to perform custom grouping + on sourcextractor++. + + :param context: + sourcextractor++ context + :param kron_scale: + Multiply the Kron radius by this factor to compute the bounding sphere + :param distance_threshold: + Distance threshold for AgglomerativeClustering. 'auto' uses twice the mean radius as an approximation. + """ + + def __init__(self, context: pipeline.Context, kron_scale: float = 2.5, + distance_threshold: Union[float, str] = 'auto'): + self.__context = context + self.__kron_scale = kron_scale + self.__distance_threshold = distance_threshold + self.__to_group = [] + + def set_next_stage(self, next_stage): + """ + Set the following stage, which must receive groups + """ + self.__next = next_stage + + def __filter_sources(self, event: pipeline.ProcessSourcesEvent): + """ + Return a list of sources to flush, and sources to keep + """ + flush, no_flush = [], [] + for source in self.__to_group: + if event.must_process(source): + flush.append(source) + else: + no_flush.append(source) + return flush, no_flush + + def __group(self, sources: List[pipeline.Source]): + """ + Group the given list of sources via AgglomerativeClustering + :return: + A list of Groups + """ + if not sources: + return [] + + logger.info('Grouping %d sources', len(sources)) + + coordinates = np.asarray([(source.pixel_centroid_x, source.pixel_centroid_y) for source in sources]) + radius = 2. * np.asarray([source.kron_radius * self.__kron_scale for source in sources]) + + if self.__distance_threshold == 'auto': + threshold = np.mean(radius) + else: + threshold = self.__distance_threshold + + A = radius_neighbors_graph(coordinates, radius=radius, mode='connectivity', include_self=True) + clustering = AgglomerativeClustering(connectivity=A, n_clusters=None, distance_threshold=threshold) + clusters = clustering.fit_predict(coordinates) + + logger.info('Found %d groups', clusters.max()) + + groups = [[] for _ in range(len(clusters))] + for i, cluster_id in enumerate(clusters): + groups[cluster_id].append(sources[i]) + + groups = list(map(lambda group: pipeline.Group.create(self.__context, group), groups)) + return groups + + def __flush(self, event: pipeline.ProcessSourcesEvent): + """ + Group and flush the sources kept in memory + """ + flusheable, self.__to_group = self.__filter_sources(event) + groups = self.__group(flusheable) + for group in groups: + self.__next(group) + # We need to forward the event + self.__next(event) + + def __call__(self, obj): + """ + Called by segmentation. Use ProcessSourceEvent.must_source to see if stored sources must be + grouped and flushed. + :param obj: + Either a ProcessSourceEvent sent by Segmentation to trigger a flush, or a + segmented source + """ + if isinstance(obj, pipeline.ProcessSourcesEvent): + self.__flush(obj) + else: + self.__to_group.append(obj) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str): + """ + Setup the sourcextractor++ pipeline with a custom grouping stage + """ + config['output-catalog-filename'] = output_path + kron_scale = float(config.get('auto-kron-factor', 2.5)) + distance_threshold = config.pop('grouping-distance-threshold', 'auto') + if distance_threshold != 'auto': + distance_threshold = float(distance_threshold) + context = pipeline.Context(config) + # Generate out custom grouping + grouping = CustomGrouping(context, kron_scale, distance_threshold) + with context: + stages = [pipeline.Segmentation(), pipeline.Partition(), grouping, pipeline.Deblending(), + pipeline.Measurement(), pipeline.FitsOutput()] + pipe = pipeline.Pipeline(stages) + pipe().get() + print(f'Done!') + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +if __name__ == '__main__': + err_handler = logging.StreamHandler(sys.stderr) + err_handler.setLevel(logging.INFO) + logger.addHandler(err_handler) + logger.setLevel(logging.INFO) + + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='FITS', default='output.fits', help='Output file') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file) From c344b90724ec425a434e5fbc9b1d215fb696a543 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 17 Oct 2022 13:23:14 +0200 Subject: [PATCH 32/35] Do not capture python stdout and stderr When calling sourcextractor++ from Python --- .../Configuration/PythonConfig.h | 13 ++--- .../PythonConfig/PythonInterpreter.h | 33 +++++++------ .../src/lib/Configuration/PythonConfig.cpp | 16 ++++-- .../lib/PythonConfig/PythonInterpreter.cpp | 49 +++++++++++-------- SEPythonModule/src/lib/Context.cpp | 7 ++- 5 files changed, 70 insertions(+), 48 deletions(-) diff --git a/SEImplementation/SEImplementation/Configuration/PythonConfig.h b/SEImplementation/SEImplementation/Configuration/PythonConfig.h index 0177bd474..ec207ddd4 100644 --- a/SEImplementation/SEImplementation/Configuration/PythonConfig.h +++ b/SEImplementation/SEImplementation/Configuration/PythonConfig.h @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -15,7 +15,7 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PythonConfig.h * @author Nikolaos Apostolakos */ @@ -29,20 +29,21 @@ namespace SourceXtractor { class PythonConfig : public Euclid::Configuration::Configuration { - + public: - + explicit PythonConfig(long manager_id); - + std::map getProgramOptions() override; void preInitialize(const UserValues& args) override; void initialize(const UserValues& args) override; - + PythonInterpreter& getInterpreter() const; private: + bool m_capture_output = true; boost::python::object m_measurement_config; }; diff --git a/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h b/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h index 3c4a4ed92..b69a77538 100644 --- a/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h +++ b/SEImplementation/SEImplementation/PythonConfig/PythonInterpreter.h @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -15,7 +15,7 @@ * along with this library; if not, write to the Free Software Foundation, Inc., * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -/* +/* * @file PythonInterpreter.h * @author Nikolaos Apostolakos */ @@ -34,17 +34,20 @@ namespace SourceXtractor { class PythonInterpreter { - + public: - + static PythonInterpreter& getSingleton(); - + void runFile(const std::string& filename, const std::vector& argv); + /// Capture Python's stdout and stderr and pass the output through the logging + void captureOutput(); + void setupContext(boost::python::object config = {}); virtual ~PythonInterpreter(); - + std::map getMeasurementImages(); std::map getApertures(); @@ -54,23 +57,23 @@ class PythonInterpreter { std::map> getApertureOutputColumns(); std::map getConstantParameters(); - + std::map getFreeParameters(); - + std::map getDependentParameters(); - + std::map getPriors(); - + std::map getConstantModels(); std::map getPointSourceModels(); - + std::map getSersicModels(); - + std::map getExponentialModels(); - + std::map getDeVaucouleursModels(); - + std::map getOnnxModels(); std::map> getFrameModelsMap(); @@ -78,7 +81,7 @@ class PythonInterpreter { std::map getModelFittingParams(); private: - + PythonInterpreter(); std::map getMapFromDict(const char* object_name, diff --git a/SEImplementation/src/lib/Configuration/PythonConfig.cpp b/SEImplementation/src/lib/Configuration/PythonConfig.cpp index 1129420d0..30d09e184 100644 --- a/SEImplementation/src/lib/Configuration/PythonConfig.cpp +++ b/SEImplementation/src/lib/Configuration/PythonConfig.cpp @@ -1,4 +1,4 @@ -/* +/** * Copyright © 2019-2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université * * This library is free software; you can redistribute it and/or modify it under @@ -31,7 +31,9 @@ namespace { const std::string PYTHON_CONFIG_FILE{"python-config-file"}; const std::string PYTHON_ARGV{"python-arg"}; +// These are internal and not exposed to the configuration manager const std::string PYTHON_CONFIG_OBJ{"python-config-object"}; +const std::string PYTHON_CAPTURE_OUTPUT{"python-capture-output"}; } // namespace @@ -58,13 +60,19 @@ void PythonConfig::preInitialize(const UserValues& args) { } else if (!filename.empty() && !fs::exists(filename)) { throw Elements::Exception() << "Python configuration file " << filename << " does not exist"; } + + auto py_capture_output = args.find(PYTHON_CAPTURE_OUTPUT); + if (py_capture_output != args.end()) { + m_capture_output = py_capture_output->second.as(); + } } void PythonConfig::initialize(const UserValues& args) { auto& singleton = PythonInterpreter::getSingleton(); - if (m_measurement_config) { - singleton.setupContext(m_measurement_config); - } else { + if (m_capture_output) { + singleton.captureOutput(); + } + if (!m_measurement_config) { auto filename = args.find(PYTHON_CONFIG_FILE)->second.as(); if (!filename.empty()) { std::vector argv; diff --git a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp index c56a11fe6..c65b46c28 100644 --- a/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp +++ b/SEImplementation/src/lib/PythonConfig/PythonInterpreter.cpp @@ -51,23 +51,31 @@ PythonInterpreter& PythonInterpreter::getSingleton() { PythonInterpreter::PythonInterpreter() : m_out_wrapper(stdout_logger), m_err_wrapper(stderr_logger) { // We may be called *from* Python! - if (Py_IsInitialized()) { - return; - } - // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt - // Here we are not interested on this behaviour, so we get whatever handler we've got (normally - // the default one) and restore it after initializing the interpreter - struct sigaction sigint_handler; - sigaction(SIGINT, nullptr, &sigint_handler); - - PyImport_AppendInittab("pyston", PYSTON_MODULE_INIT); - Py_Initialize(); + if (!Py_IsInitialized()) { + struct sigaction sigint_handler; + + // Python sets its own signal handler for SIGINT (Ctrl+C), so it can throw a KeyboardInterrupt + // Here we are not interested on this behaviour, so we get whatever handler we've got (normally + // the default one) and restore it after initializing the interpreter + sigaction(SIGINT, nullptr, &sigint_handler); + + // Make pyston importable + PyImport_AppendInittab("pyston", PYSTON_MODULE_INIT); + + // Initialize Python + Py_Initialize(); #if PY_VERSION_HEX < 3090000 - PyEval_InitThreads(); + PyEval_InitThreads(); #endif - PyEval_SaveThread(); + PyEval_SaveThread(); + + // Restore SIGINT handler + sigaction(SIGINT, &sigint_handler, nullptr); + } - sigaction(SIGINT, &sigint_handler, nullptr); + // Import ourselves so the conversions are registered + Pyston::GILLocker locker; + py::import("_SEPythonConfig"); } PythonInterpreter::~PythonInterpreter() { @@ -97,13 +105,6 @@ void PythonInterpreter::runFile(const std::string& filename, const std::vectorinitialize(options); m_segmentation_factory->configure(*m_config_manager); From 1f43d9edbf496d238f348120baf23d6defdffe9c Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Mon, 17 Oct 2022 14:07:18 +0200 Subject: [PATCH 33/35] Restore backtrace when raising Python exceptions i.e. Python => C++ => Python --- CMakeLists.txt | 2 +- SEPythonModule/src/lib/SePyMain.cpp | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a497f1e6a..383cca660 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,4 +6,4 @@ find_package(ElementsProject) #--------------------------------------------------------------- # Declare project name and version -elements_project(SourceXtractorPlusPlus 0.19 USE Alexandria 2.27 DESCRIPTION "SourceXtractor++, the next generation SExtractor") +elements_project(SourceXtractorPlusPlus 0.19 USE Alexandria 2.28 DESCRIPTION "SourceXtractor++, the next generation SExtractor") diff --git a/SEPythonModule/src/lib/SePyMain.cpp b/SEPythonModule/src/lib/SePyMain.cpp index 7d6dafd77..fb0d2f805 100644 --- a/SEPythonModule/src/lib/SePyMain.cpp +++ b/SEPythonModule/src/lib/SePyMain.cpp @@ -104,6 +104,10 @@ struct std_duration_from_timedelta { } }; +void translate_pyston_exc(const Pyston::Exception& exc) { + exc.restore(); +} + } // namespace BOOST_PYTHON_MODULE(_SEPythonModule) { @@ -206,6 +210,9 @@ BOOST_PYTHON_MODULE(_SEPythonModule) { // For custom segmentation py::def("AllFramesDone", &AllFramesDone::create); + // Pyston::Exception wrap Python errors, so unwrap them at the Python boundary + py::register_exception_translator(&translate_pyston_exc); + // Import pyston into the interpreter so it is importable without tweaking PYTHONPATH #if PY_MAJOR_VERSION >= 3 PyObject* pyston = PyInit_pyston(); From d2c2be924e987fb161c52236762c25a7a5669a80 Mon Sep 17 00:00:00 2001 From: Alejandro Alvarez Ayllon Date: Thu, 20 Oct 2022 15:52:22 +0200 Subject: [PATCH 34/35] Add demo with live preview --- SEPythonWrapper/scripts/LivePreviewDemo.py | 123 +++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 SEPythonWrapper/scripts/LivePreviewDemo.py diff --git a/SEPythonWrapper/scripts/LivePreviewDemo.py b/SEPythonWrapper/scripts/LivePreviewDemo.py new file mode 100644 index 000000000..239d49750 --- /dev/null +++ b/SEPythonWrapper/scripts/LivePreviewDemo.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2022 Université de Genève, LMU Munich - Faculty of Physics, IAP-CNRS/Sorbonne Université +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 3.0 of the License, or (at your option) +# any later version. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +import itertools +import threading +from argparse import ArgumentParser +from configparser import ConfigParser +from typing import Any, Dict + +import matplotlib.pyplot as plt +from astropy.io import fits +from matplotlib.animation import FuncAnimation +from matplotlib.colors import SymLogNorm +from sourcextractor import __version__ as seversion +from sourcextractor import pipeline + + +class LivePreview: + """ + Display sources on a matplotlib figure as they are being detected + """ + + def __init__(self, image): + self.__next_stage = None + self.__figure, self.__ax = plt.subplots() + self.__height, self.__width = fits.open(image)[0].data.shape + self.__ani = FuncAnimation(self.__figure, self.__update, interval=1000, init_func=self.__init_fig, blit=False) + self.__artists = [] + self.__norm = SymLogNorm(10) + + def set_next_stage(self, next_stage): + self.__next_stage = next_stage + + def __init_fig(self): + self.__ax.set_xlim(0, self.__width) + self.__ax.set_ylim(0, self.__height) + return [] + + def __update(self, frame: int): + artists = self.__artists + self.__artists = [] + return artists + + def __add_stamp(self, source): + stamp = source.detection_filtered_stamp + pos_x, pos_y = source.pixel_centroid_x, source.pixel_centroid_y + left, right = pos_x - stamp.shape[1] // 2, pos_x + stamp.shape[1] // 2 + bottom, top = pos_y - stamp.shape[0] // 2, pos_y + stamp.shape[1] // 2 + self.__artists.append( + self.__ax.imshow(stamp, origin='lower', extent=(left, right, bottom, top), cmap='Greys_r', norm=self.__norm) + ) + + def show(self): + self.__figure.show() + + def stop(self): + self.__ani.event_source.stop() + + def __call__(self, obj): + if isinstance(obj, pipeline.Source): + self.__add_stamp(obj) + elif isinstance(obj, pipeline.Group): + [self.__add_stamp(source) for source in obj] + self.__next_stage(obj) + + +def run_sourcextractor(config: Dict[str, Any], output_path: str): + """ + Setup the sourcextractor++ pipeline + """ + config['output-catalog-filename'] = output_path + live = LivePreview(config['detection-image']) + with pipeline.Context(config): + stages = [pipeline.Segmentation(), pipeline.Partition(), + pipeline.Grouping(), pipeline.Deblending(), live, pipeline.Measurement(), pipeline.FitsOutput()] + pipe = pipeline.Pipeline(stages) + + # UI must run on the main thread, so sourcextractor++ must run on another + sourcex_thread = threading.Thread(target=lambda: pipe().get()) + sourcex_thread.start() + while sourcex_thread.is_alive(): + live.show() + plt.pause(0.05) + sourcex_thread.join() + print(f'Done!') + live.stop() + plt.show(block=True) + + +def parse_config_file(path: str) -> Dict[str, Any]: + """ + Parse a sourcextractor++ (like) config file into a dictionary + """ + parser = ConfigParser() + with open(path, 'rt') as fd: + parser.read_file(itertools.chain(['[main]'], fd)) + return {k: v for k, v in parser.items('main')} + + +if __name__ == '__main__': + print(f'Running sourcextractor++ {seversion}') + + parser = ArgumentParser() + parser.add_argument('--output-file', type=str, metavar='FITS', default='output.fits', help='Output file') + parser.add_argument('config_file', type=str, metavar='CONFIGURATION', help='Configuration file') + + args = parser.parse_args() + run_sourcextractor(parse_config_file(args.config_file), args.output_file) From a686618cd563ce5bb8c0f9628cd1055e256bc028 Mon Sep 17 00:00:00 2001 From: Marc Schefer Date: Fri, 13 Jun 2025 14:35:09 +0200 Subject: [PATCH 35/35] fix compilation error --- SEPythonModule/src/lib/Grouping.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SEPythonModule/src/lib/Grouping.cpp b/SEPythonModule/src/lib/Grouping.cpp index 2e94e1851..2675c25e1 100644 --- a/SEPythonModule/src/lib/Grouping.cpp +++ b/SEPythonModule/src/lib/Grouping.cpp @@ -27,7 +27,7 @@ namespace se = SourceXtractor; namespace SourceXPy { Grouping::Grouping(ContextPtr context) : m_context(std::move(context)) { - m_grouping = m_context->m_grouping_factory->createGrouping(); + m_grouping = std::dynamic_pointer_cast(m_context->m_grouping_factory->createGrouping()); } std::string Grouping::repr() const {