[wip] documentation of exports

source/_static/code/exports/BirthdateFilter.php

@@ -0,0 +1,123 @@
+<?php
+
+namespace Chill\PersonBundle\Export\Filter;
+
+use Chill\MainBundle\Export\FilterInterface;
+use Symfony\Component\Form\Extension\Core\Type\DateType;
+use Symfony\Component\Validator\Context\ExecutionContextInterface;
+use Symfony\Component\Validator\Constraints;
+use Symfony\Component\Form\FormEvent;
+use Symfony\Component\Form\FormEvents;
+use Doctrine\ORM\Query\Expr;
+use Chill\MainBundle\Form\Type\Export\FilterType;
+use Symfony\Component\Form\FormError;
+use Chill\MainBundle\Export\ExportElementValidatedInterface;
+
+class BirthdateFilter implements FilterInterface, ExportElementValidatedInterface
+{
+    // add specific role for this filter    
+    public function addRole()
+    {
+        // we do not need any new role for this filter, so we return null
+        return null;
+    }
+
+    // we give information on which type of export this filter applies
+    public function applyOn()
+    {
+        return 'person';
+    }
+
+    public function getTitle()
+    {
+        return 'Filter by person\'s birthdate';
+    }
+
+    // we build a form to collect some parameters from the users
+    public function buildForm(\Symfony\Component\Form\FormBuilderInterface $builder)
+    {
+        $builder->add('date_from', DateType::class, array(
+            'label' => "Born after this date",
+            'data'  => new \DateTime(),
+            'attr'  => array('class' => 'datepicker'),
+            'widget'=> 'single_text',
+            'format' => 'dd-MM-yyyy',
+        ));
+        
+        $builder->add('date_to', DateType::class, array(
+            'label' => "Born before this date",
+            'data'  => new \DateTime(),
+            'attr'  => array('class' => 'datepicker'),
+            'widget'=> 'single_text',
+            'format' => 'dd-MM-yyyy',
+        ));
+        
+    }
+   
+    // the form created above must be validated. The process of validation
+    // is executed here. This function is added by the interface 
+    // `ExportElementValidatedInterface`, and can be ignore if there is 
+    // no need for a validation
+    public function validateForm($data, ExecutionContextInterface $context)
+    { 
+        $date_from = $data['date_from'];
+        $date_to   = $data['date_to'];
+        
+        if ($date_from === null) {
+            $context->buildViolation('The "date from" should not be empty')
+                //->atPath('date_from')
+                ->addViolation();
+        }
+        
+        if ($date_to === null) {
+            $context->buildViolation('The "date to" should not be empty')
+                //->atPath('date_to')
+                ->addViolation();
+        }
+        
+        if (
+            ($date_from !== null && $date_to !== null)
+            &&
+            $date_from >= $date_to
+        ) {
+            $context->buildViolation('The date "date to" should be after the '
+                . 'date given in "date from" field')
+                ->addViolation();
+        }
+    }
+
+
+    // here, we alter the query created by Export
+    public function alterQuery(\Doctrine\ORM\QueryBuilder $qb, $data)
+    {
+        $where = $qb->getDQLPart('where');
+        // we create the clause here
+        $clause = $qb->expr()->between('person.birthdate', ':date_from', 
+            ':date_to');
+
+        // we have to take care **not to** remove previous clauses...
+        if ($where instanceof Expr\Andx) {
+            $where->add($clause);
+        } else {
+            $where = $qb->expr()->andX($clause);
+        }
+        
+        $qb->add('where', $where);
+        // we add parameters from $data. $data contains the parameters from the form
+        $qb->setParameter('date_from', $data['date_from']);
+        $qb->setParameter('date_to', $data['date_to']);
+    }
+
+    // here, we create a simple string which will describe the action of
+    // the filter in the Response
+    public function describeAction($data, $format = 'string')
+    {
+        return array('Filtered by person\'s birtdate: '
+            . 'between %date_from% and %date_to%', array(
+                '%date_from%' => $data['date_from']->format('d-m-Y'),
+                '%date_to%'   => $data['date_to']->format('d-m-Y')
+            ));
+    }
+
+
+}

source/_static/code/exports/CountPerson.php

@@ -0,0 +1,118 @@
+<?php
+
+
+namespace Chill\PersonBundle\Export\Export;
+
+use Chill\MainBundle\Export\ExportInterface;
+use Doctrine\ORM\QueryBuilder;
+use Symfony\Component\Form\FormBuilderInterface;
+use Doctrine\ORM\Query;
+use Chill\PersonBundle\Security\Authorization\PersonVoter;
+use Symfony\Component\Security\Core\Role\Role;
+use Chill\PersonBundle\Export\Declarations;
+use Chill\MainBundle\Export\FormatterInterface;
+use Doctrine\ORM\EntityManagerInterface;
+
+/**
+ * 
+ *
+ * @author Julien Fastré <julien.fastre@champs-libres.coop>
+ */
+class CountPerson implements ExportInterface
+{
+    /**
+     *
+     * @var EntityManagerInterface
+     */
+    protected $entityManager;
+    
+    public function __construct(
+            EntityManagerInterface $em
+            )
+    {
+        $this->entityManager = $em;
+    }
+    
+    public function getType()
+    {
+        return Declarations::PERSON_TYPE;
+    }
+    
+    public function getDescription()
+    {
+        return "Count peoples by various parameters.";
+    }
+    
+    public function getTitle()
+    {
+        return "Count peoples";
+    }
+    
+    public function requiredRole()
+    {
+        return new Role(PersonVoter::STATS);
+    }
+    
+    public function initiateQuery(array $requiredModifiers,  array $acl, array $data = array())
+    {
+        // we gather all center the user choose.
+        $centers = array_map(function($el) { return $el['center']; }, $acl);
+        
+        $qb = $this->entityManager->createQueryBuilder();
+        
+        $qb->select('COUNT(person.id) AS export_result')
+                ->from('ChillPersonBundle:Person', 'person')
+                ->join('person.center', 'center')
+                ->andWhere('center IN (:authorized_centers)')
+                ->setParameter('authorized_centers', $centers);
+                ;
+        
+        
+        return $qb;
+    }
+    
+    public function getResult($qb, $data)
+    {
+        return $qb->getQuery()->getResult(Query::HYDRATE_SCALAR);
+    }
+    
+    public function getQueryKeys($data)
+    {
+        // this array match the result keys in the query. We have only
+        // one column.
+        return array('export_result');
+    }
+    
+    public function getLabels($key, array $values, $data)
+    {
+        
+        // the Closure which will be executed by the formatter. 
+        return function($value) {
+            switch($value) {
+                case '_header':
+                  // we have to process specifically the '_header' string, 
+                  // which will be used by the formatter to show a column title
+                  return $this->getTitle();
+                default:
+                  // for all value, we do not process them and return them
+                  // immediatly
+                  return $value;
+        };
+    }
+    
+    public function getAllowedFormattersTypes()
+    {
+        return array(FormatterInterface::TYPE_TABULAR);
+    }
+    
+    public function buildForm(FormBuilderInterface $builder) {
+        // this export does not add any form
+    }
+    
+    public function supportsModifiers()
+    {
+        // explain the export manager which formatters and filters are allowed
+        return array(Declarations::PERSON_TYPE, Declarations::PERSON_IMPLIED_IN);
+    }
+    
+}

source/_static/puml/exports/form_steps.puml

@@ -0,0 +1,82 @@
+@startuml
+
+actor User
+participant Controller
+participant ExportFormType as eft
+participant ExportManager as em
+participant SelectedExport as se
+participant AggregatorFormType as aft
+collections aggregators as a
+participant FilterFormType as fft
+collections filters as f
+
+
+User -> Controller: request "/exports/new/<id of the export>?step=export"
+
+activate Controller
+Controller -> eft: build the form (AbstractType::buildForm)
+
+activate eft
+eft -> em: get the export
+
+activate em
+em -> eft: return the SelectedExport
+deactivate em
+
+eft -> se: get the Export Type
+
+activate se
+se -> eft: return the export Type (a string)
+deactivate se
+
+eft -> aft: build the subform 'aggregators'
+activate aft
+
+aft -> em: get aggregators for this export type
+activate em
+em -> aft: return a collection of aggregators
+deactivate em
+
+loop for each aggregator
+  aft -> a: build eventual subform for the aggregator 
+  activate a
+  a -> a: append enventually his form
+  a -> aft
+  deactivate a
+end
+
+aft -> eft
+deactivate aft
+
+eft -> fft: build the subform 'filters'
+activate fft 
+
+fft -> em: get filters for this export type
+activate em
+em -> fft: return a collection for filters
+deactivate em
+
+loop for each filter
+  fft -> f: build eventual subform for the filter
+  activate f
+  f -> f: append eventually his form
+  f -> fft
+  deactivate f
+end
+
+fft -> eft
+deactivate fft
+
+eft -> se: build eventual subform for the export itsefl
+activate se
+se -> se: append eventually his form
+se -> eft 
+deactivate se
+
+se -> Controller: return a well-build form
+deactivate se
+
+Controller -> User: render the page with the form
+deactivate Controller
+@enduml
+

source/_static/puml/exports/processing_export.puml

@@ -0,0 +1,164 @@
+@startuml
+
+participant Controller
+participant ExportManager as em
+participant SelectedExport as se
+participant Formatter as f
+collections aggregators as aa
+collections filters as ff
+
+activate Controller
+Controller -> em: ExportManager::generate(string 'export_alias', Centers[], $data, $formatterdata)
+activate em
+note left
+  $data contains the data 
+     from the export form 
+     (export form + aggregators form + 
+     filters forms)
+end note
+
+
+em -> se: Export::initiateQuery($modifiers, $acl, $data)
+activate se
+note left
+  $modifiers contains the selected modifiers
+    and their data. Usually you can ignore this.
+  $acl contains the list of selected centers. The
+    export must check that the user has the right 
+    to compute the export for this center
+  $data the data from the export form (if the export
+    build a form with `buildForm` function
+end note
+
+create Query as q
+se -> q: the export will create a query
+se -> em: return the query
+deactivate se
+
+alt The query is a "native sql query"
+ 
+  note over em, se
+    When the export create a native query, there aren't any filters
+    or aggregators which can alter the query. 
+
+    We immediatly get the results from the query.
+  end note
+ 
+else "The query is a query builder"
+
+  note over em, se
+    As the query is a query builder, filters and formatters 
+    will be able to alter the query. 
+  end note
+
+  loop for each filter selected by user
+    em -> ff: FilterInterface::alterQuery(query, $data)
+    activate ff
+    note over em, ff
+      The filter will alter query, adding his own clause (i.e. WHERE clause). It is 
+      his responsability to avoid removing clauses added by other filters / aggregators.
+
+      The ExportManager will also transmit the $data filled by the user (if the 
+      filter has a form) to each filter / aggregator.
+    end note
+    ff -> q: append some clauses
+    deactivate ff
+  end
+
+  loop for each aggregator selected by user
+    note over em, aa
+      As of filter, aggregators will append their
+      own clauses in the query (usually GROUP BY clause).
+    end note
+    em -> aa: AggregatorInterface::alterQuery(query, data)
+    activate aa
+
+    aa -> q: append some clauses
+    deactivate aa
+  end
+end alt
+
+note over se
+  The query is now ready. We will fetch the results from databases.
+
+  The Export is responsible for getting the results
+end note 
+
+em -> se: Export::getResult(query, $data)
+activate se
+
+se -> q: getResult()
+activate q
+q -> se: return result
+destroy q
+
+se -> em: return result
+deactivate se
+
+em -> f: FormatterInterface::getResponse()
+activate f
+
+note over f, ff
+  The formatter will ask the export, and each aggregators the keys they
+  are responsible for. 
+
+  Then, for each of those keys, he will ask to each participant 
+  (Export or aggregator) a Closure, which will render the value in 
+  results into a human readable string.
+end note
+
+f -> se: getQueryKeys
+activate se
+se -> f: return string[]
+
+loop for each keys the export is responsible for
+  f -> se: ExportInterface::getLabel()
+  create "closure for key export" as closuree
+  se -> closuree: create a closure
+  se -> f: return closure
+end
+
+loop for each aggregator
+  f -> aa: getQueryKeys()
+  activate aa
+  aa -> f: return string[]
+  deactivate aa
+
+  loop for each keys the aggregator is responsible for
+    f -> aa: getLabel()
+    activate aa
+    create "closure for key aggregators" as closureg
+    aa -> closureg: create a closure
+    aa -> f: return closure
+    deactivate aa
+  end
+end 
+
+loop over results
+  note over f, closureg
+    Each row in result will be transformed in a human readable format by the closure.
+
+    The human readable string will be appended by the Formatter in his response (a Spreadsheet, CSV file, ...)
+  end note
+
+  f -> closuree: call
+  activate closuree
+  closuree -> f: return a human readable format for this value
+  deactivate closuree
+
+  f -> closureg: call
+  activate closureg
+  closureg -> f: return a human readable format for the value
+  deactivate g
+
+  f -> f: append the result in his response
+end 
+
+f -> em: return a Response
+deactivate f
+
+em -> Controller: return a Response
+deactivate em
+deactivate Controller
+
+@enduml

source/development/exports.rst

@@ -32,7 +32,7 @@ Concepts
 Some vocabulary: 3 "Export elements"
 ------------------------------------
 
-Three terms are used for this framework : 
+Four terms are used for this framework : 
 
 exports
    provides some basic operation on the date. Two kind of exports are available : 
@@ -50,13 +50,17 @@ aggregators
 
    Example of aggregator : "group people by gender", "group people by nationality", "group activity by type", ...
 
+formatters
+   The formatters format the data into a :class:`Symfony\Component\HttpFoundation\Response`, which  will be returned "as is" by the controller to the web client. 
+
+   Example of formatter: "format data as CSV", "format data as ods spreadsheet", ...
 
 Anatomy of an export
 ---------------------
 
 An export may be explained as a sentence, where each part of this sentence refers to one or multiple exports element. Examples : 
 
-**Example 1**: Count the number of people having at least one activity in the last 12 month, and group them by nationality and gender
+**Example 1**: Count the number of people having at least one activity in the last 12 month, and group them by nationality and gender, and format them in a CSV spreadsheet.
 
 Here : 
 
@@ -64,10 +68,11 @@ Here :
 - *having at least one activity* is the filter part
 - *group them by nationality* is the aggregator part
 - *group them by gender* is a second aggregator part
+- *format the date in a CSV spreadsheet* is the formatter part
 
 Note that :
 
-- aggregators, filters and exports are cross-bundle. Here the bundle activity provides a filter which apply on an export provided by the person bundle ;
+- aggregators, filters, exports and aggregators are cross-bundle. Here the bundle *activity* provides a filter which apply on an export provided by the person bundle ;
 - there may exists multiple aggregator or filter for one export. Currently, only one export is allowed.
 
 The result might be : 
@@ -84,7 +89,7 @@ The result might be :
 | France                | Female         | 150                       |
 +-----------------------+----------------+---------------------------+
 
-**Example 2**: Count the average duration of an activity with type "meeting", which occurs between the 1st of June and the 31st of December, and group them by week.
+**Example 2**: Count the average duration of an activity with type "meeting", which occurs between the 1st of June and the 31st of December, group them by week, and format the data in a OpenDocument spreadsheet.
 
 Here : 
 
@@ -92,6 +97,7 @@ Here :
 - *activity with type meeting* is a filter part
 - *activity which occurs between the 1st of June and the 31st of December* is a filter
 - *group them by week* is the aggregator part
+- *format the date in an OpenDocument spreadsheet* is the formatter part
 
 The result might be :
 
@@ -116,8 +122,8 @@ In other words, developers are required to take care of user authorization for e
 
 It should exists a special role that should be granted to users which are allowed to build exports. For more simplicity, this role should apply on center, and should not requires special circles.
 
-How the magic works
-===================
+How does the magic works ?
+===========================
 
 To build an export, we rely on the capacity of the database to execute queries with aggregate (i.e. GROUP BY) and filter (i.e. WHERE) instructions.
 
@@ -125,13 +131,111 @@ An export is an SQL query which is initiated by an export, and modified by aggre
 
 .. note:: 
 
-   **Example 1**: Count the number of people having at least one activity in the last 12 month, and group them by nationality and gender
+   **Example**: Count the number of people having at least one activity in the last 12 month, and group them by nationality and gender
 
-   1. The report initiate the query :code:`SELECT count(people.*) FROM people`
-   2. The filter add a where and join clause : :code:`SELECT count(people.*) FROM people RIGHT JOIN activity WHERE activity.date IS BETWEEN now AND 6 month ago`
-   3. The aggregator "nationality" add a GROUP BY clause and a column in the SELECT statement: :code:`SELECT people.nationality, count(people.*) FROM people RIGHT JOIN activity WHERE activity.date IS BETWEEN now AND 6 month ago GROUP BY nationality`
-   4. The aggregator "gender" do the same job as the nationality aggregator : it adds a GROUP BY clause and a column in the SELECT statement : :code:`SELECT people.nationality, people.gender, count(people.*) FROM people RIGHT JOIN activity WHERE activity.date IS BETWEEN now AND 6 month ago GROUP BY nationality, gender`
+   1. The report initiate the query 
 
+   .. code-block:: SQL
+
+      SELECT count(people.*) FROM people
+
+   2. The filter add a where and join clause : 
+
+   .. code-block:: SQL
+
+      SELECT count(people.*) FROM people 
+         RIGHT JOIN activity 
+         WHERE activity.date IS BETWEEN now AND 6 month ago
+
+   3. The aggregator "nationality" add a GROUP BY clause and a column in the SELECT statement: 
+
+   .. code-block:: sql
+
+      SELECT people.nationality, count(people.*) FROM people 
+         RIGHT JOIN activity 
+         WHERE activity.date IS BETWEEN now AND 6 month ago 
+         GROUP BY nationality
+
+   4. The aggregator "gender" do the same job as the nationality aggregator : it adds a GROUP BY clause and a column in the SELECT statement : 
+
+   .. code-block:: sql
+
+      SELECT people.nationality, people.gender, count(people.*) 
+         FROM people RIGHT JOIN activity 
+         WHERE activity.date IS BETWEEN now AND 6 month ago 
+         GROUP BY nationality, gender
+
+Each filter, aggregator and filter may collect parameters from the user by providing a form. This form is appended to the export form. Here is an example.
+
+.. figure:: /_static/screenshots/development/export_form-fullpage.png
+
+   The screenshot show the export form for ``CountPeople`` (Nombre de personnes). The filter by date of birth is checked (*Filtrer par date de naissance de la personne*), which allow to show a subform, which is provided by the :class:`Chill\PersonBundle\Export\Filter\BirthdateFilter`. The other filter, which are unchecked, does not show the subform.
+
+   Two aggregators are also checked : by Country of birth (*Aggréger les personnes par pays de naissance*, corresponding class is :class:`Chill\PersonBundle\Export\Aggregator\CountryOfBirthAggregator`, which also open a subform. The aggregator by gender (*Aggréger les personnes par genre*) is also checked, but there is no corresponding subform.
+
+The Export Manager
+------------------
+
+The Export manager (:class:`Chill\MainBundle\Export\ExportManager` is the central class which register all exports, aggregators, filters and formatters.
+
+The export manager is also responsible for orchestrating the whole export process, producing a :class:`Symfony\FrameworkBundle\HttpFoundation\Request` to each export request.
+
+
+The export form step
+--------------------
+
+The form step allow to build a form, aggregating different parts of the module.
+
+The building of forms is separated between different subform, which are responsible for rendering their part of the form (aggregators, filters, and export). 
+
+.. figure:: /_static/puml/exports/form_steps.png
+   :scale: 40%
+
+The formatter form step
+-----------------------
+
+The formatter form is processed *after* the user filled the export form. It is built the same way, but receive in parameters the data entered by the user on the previous step (i.e. export form). It may then adapt it accordingly (example: show a list of columns selected in aggregators).
+
+Processing the export
+---------------------
+
+The export process may be explained by this schema :
+
+.. figure:: /_static/puml/exports/processing_export.png
+   :scale: 40%
+
+   (Click to enlarge)
+
+
+Export, formatters and filters explained
+========================================
+
+Exports
+-------
+
+This is an example of the ``CountPerson`` export :
+
+.. literalinclude:: /_static/code/exports/CountPerson.php
+   :language: php
+   :linenos:
+
+* **Line 36**: the ``getType`` function return a string. This string will be used to find the aggregtors and filters which will apply to this export.
+* **Line 41**: a simple description to help user to understand what your export does.
+* **Line 46**: The title of the export. A summary of what your export does.
+* **Line 51**: The list of roles requires to execute this export.
+* **Line 56**: We initiate the query here...
+* **Line 59**: We have to filter the query with centers the users checked in the form. We process the $acl variable to get all ``Center`` object in one array
+* **Line 63**: We create the query, with a query builder.
+* **Line 74**: We simply returns the result, but take care of hydrating the results as an array.
+* **Line 103**: return the list of formatters types which are allowed to apply on this filter
+
+Filters
+-------
+
+This is an example of the *filter by birthdate*. This filter ask some information in a form (`buildForm` is not empty), and this form must be validated. To performs this validations, we implement a new Interface: :class:`Chill\MainBundle\Export\ExportElementValidatedInterface`:
+
+.. literalinclude:: /_static/code/exports/BirthdateFilter.php
+   :language: php
 
 .. todo::