Merge branch 'master' into types

21 files changed, 683 insertions, 0 deletions

diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000..3274a8a
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,73 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'lesana'
+copyright = "2020, Elena Grandi"
+author = "Elena ``of Valhalla''"
+
+# The full version, including alpha/beta/rc tags
+release = '0.6.0'
+# The major project version
+version = '0.6'
+
+# compatibility with sphinx 1.8 on buster
+master_doc = 'index'
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.viewcode',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+man_pages = [
+    ('man/lesana-edit', 'lesana-edit', '', '', 1),
+    ('man/lesana-export', 'lesana-export', '', '', 1),
+    ('man/lesana-index', 'lesana-index', '', '', 1),
+    ('man/lesana-init', 'lesana-init', '', '', 1),
+    ('man/lesana-new', 'lesana-new', '', '', 1),
+    ('man/lesana-rm', 'lesana-rm', '', '', 1),
+    ('man/lesana', 'lesana', '', '', 1),
+    ('man/lesana-search', 'lesana-search', '', '', 1),
+    ('man/lesana-show', 'lesana-show', '', '', 1),
+]
diff --git a/docs/source/getting_started_command_line.rst b/docs/source/getting_started_command_line.rst
new file mode 100644
index 0000000..0c4d507
--- /dev/null
+++ b/docs/source/getting_started_command_line.rst
@@ -0,0 +1,62 @@
+******************************
+Getting Started (Command Line)
+******************************
+
+lesana can be used from the command line through the ``lesana`` command;
+for more details run ``lesana help``.
+
+Many commands will try to open some file in an editor: they will attempt
+to use, in this order, ``$EDITOR``, ``sensible-editor`` or as a fallback
+``vi``, which should be installed on any POSIX-like system.
+
+To start a new collection, create a directory and run ``lesana
+init`` into it::
+
+    mkdir $DIRECTORY
+    cd $DIRECTORY
+    lesana init
+
+It will create the basic file structure of a lesana collection,
+including a ``settings.yaml`` skeleton and it will initialize a git
+repository (use ``--no-git`` to skip this part and ignore all further
+git commands).
+
+It will then open ``settings.yaml`` in an editor: fill in your list of
+fields and all other data, save and exit.
+You are now ready to commit the configuration for your new collection::
+
+    git commit -m 'Collection settings'
+
+An empty collection is not very interesting: let us start adding new
+entries::
+
+   lesana new
+
+It will again open an editor on a skeleton of entry where you can fill
+in the values. When you close the editor it will print the entry id,
+that you can use e.g. to edit again the same entry::
+
+   lesana edit $ENTRY_ID
+
+After you've added a few entries, you can now search for some word that
+you entered in one of the indexed fields::
+
+   lesana search some words
+
+this will also print the entry ids of matching items, so that you can
+open them with ``lesana edit``.
+
+If you're using git, entries will be autoadded to the staging area, but
+you need to commit them, so that you can choose how often you do so.
+
+Search results are limited by default to 12 matches; to get all results
+for your query you can use the option ``--all``. This is especially
+useful when passing the results to a template::
+
+   lesana search --template templates/my_report.html --all \
+       some search terms \
+       > some_search_terms-report.html
+
+will generate an html file based on the jinja2 template
+``templates/my_report.html`` with all the entries found for those search
+terms.
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000..292a0ce
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,28 @@
+.. lesana documentation master file, created by
+   sphinx-quickstart on Thu Oct  1 22:28:26 2020.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to lesana's documentation!
+==================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   getting_started_command_line
+   moving_data_between_collections
+   promises
+
+   man/index
+
+   reference/modules
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/docs/source/man/Makefile b/docs/source/man/Makefile
new file mode 100644
index 0000000..0369f59
--- /dev/null
+++ b/docs/source/man/Makefile
@@ -0,0 +1,12 @@
+
+MAN_TARGETS = $(patsubst %.rst,%.1,$(wildcard *.rst))
+
+all: $(MAN_TARGETS)
+
+%.1: %.rst
+	rst2man $< > $@
+
+clean:
+	rm *.1
+
+.PHONY: all clean
diff --git a/docs/source/man/index.rst b/docs/source/man/index.rst
new file mode 100644
index 0000000..65d92a1
--- /dev/null
+++ b/docs/source/man/index.rst
@@ -0,0 +1,17 @@
+*********
+Man Pages
+*********
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   lesana-edit
+   lesana-export
+   lesana-index
+   lesana-init
+   lesana-new
+   lesana-rm
+   lesana
+   lesana-search
+   lesana-show
diff --git a/docs/source/man/lesana-edit.rst b/docs/source/man/lesana-edit.rst
new file mode 100644
index 0000000..75bcaeb
--- /dev/null
+++ b/docs/source/man/lesana-edit.rst
@@ -0,0 +1,35 @@
+===========
+lesana-edit
+===========
+
+------------------------------
+edits an existing lesana entry
+------------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana edit [--help] [--collection <collection>] [--no-git] <entry>
+
+DESCRIPTION
+===========
+
+Lesana edit will open an existing entry (specified by id or partial id)
+in an editor, so that it can be changed.
+
+If the collection is configured to use git, after the editor has been
+closed, it will add the file to the git staging area, unless
+``--no-git`` is given.
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+--no-git
+   Don't add the new entry to git.
diff --git a/docs/source/man/lesana-export.rst b/docs/source/man/lesana-export.rst
new file mode 100644
index 0000000..6088dbf
--- /dev/null
+++ b/docs/source/man/lesana-export.rst
@@ -0,0 +1,32 @@
+=============
+lesana-export
+=============
+
+-------------------------------------------------
+export data from one lesana collection to another
+-------------------------------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana export [-h] [--collection COLLECTION] [--query QUERY] destination template
+
+DESCRIPTION
+===========
+
+Lesana export converts entries from one lesana collection to another,
+using a jinja2 template.
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+--query QUERY, -q QUERY
+   Xapian query to search in the collection
+
diff --git a/docs/source/man/lesana-index.rst b/docs/source/man/lesana-index.rst
new file mode 100644
index 0000000..d535c02
--- /dev/null
+++ b/docs/source/man/lesana-index.rst
@@ -0,0 +1,30 @@
+============
+lesana-index
+============
+
+------------------
+Index some entries
+------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana index [--help] [--collection COLLECTION] [files [files ...]]
+
+DESCRIPTION
+===========
+
+Lesana index adds some entries to the xapian cache, listed by filename
+(by default all of the files found in the items directory).
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+
diff --git a/docs/source/man/lesana-init.rst b/docs/source/man/lesana-init.rst
new file mode 100644
index 0000000..92527c0
--- /dev/null
+++ b/docs/source/man/lesana-init.rst
@@ -0,0 +1,40 @@
+===========
+lesana-init
+===========
+
+------------------------------
+initialize a lesana collection
+------------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana init [--help] [--collection <collection>] [--no-git]
+
+DESCRIPTION
+===========
+
+lesana init initializes a new lesana collection.
+
+It will create the directory (if it does not exist) and, unless
+``--no-git`` is specified it will initialize it as a git repository and
+create a ``.gitignore`` file with some relevant contents.
+
+It will then create a skeleton ``settings.yaml`` file and open it in an
+editor to start configuring the collection.
+
+When leaving the editor, again unless ``--no-git`` is used, it will add
+this file to the git staging area, but not commit it.
+
+OPTIONS
+=======
+
+--help, -h
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The directory where the collection will be initialized. Default is .
+--no-git
+   Do not use git in the current collection.
diff --git a/docs/source/man/lesana-new.rst b/docs/source/man/lesana-new.rst
new file mode 100644
index 0000000..4c7a10d
--- /dev/null
+++ b/docs/source/man/lesana-new.rst
@@ -0,0 +1,37 @@
+==========
+lesana-new
+==========
+
+-------------------------
+create a new lesana entry
+-------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana new [--help] [--collection <collection>] [--no-git]
+
+DESCRIPTION
+===========
+
+Lesana new creates a new lesana entry.
+
+It will create an empty entry and open an editor so that it can be
+filled.
+
+If the collection is configured to use git, after the editor has been
+closed, it will add the file to the git staging area, unless
+``--no-git`` is given.
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+--no-git
+   Don't add the new entry to git.
diff --git a/docs/source/man/lesana-rm.rst b/docs/source/man/lesana-rm.rst
new file mode 100644
index 0000000..4f765a5
--- /dev/null
+++ b/docs/source/man/lesana-rm.rst
@@ -0,0 +1,30 @@
+=========
+lesana-rm
+=========
+
+----------------------------------------
+remove an entry from a lesana collection
+----------------------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana rm [-h] [--collection COLLECTION] entries [entries ...]
+
+DESCRIPTION
+===========
+
+Lesana rm removes an entry from the collection, removing both the file
+and the cached entry.
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+
diff --git a/docs/source/man/lesana-search.rst b/docs/source/man/lesana-search.rst
new file mode 100644
index 0000000..9375541
--- /dev/null
+++ b/docs/source/man/lesana-search.rst
@@ -0,0 +1,37 @@
+=============
+lesana-search
+=============
+
+---------------------------------
+search inside a lesana collection
+---------------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana search [--help] [--collection COLLECTION] [--template TEMPLATE] \
+[--offset OFFSET] [--pagesize PAGESIZE] [--all] query [query ...]
+
+DESCRIPTION
+===========
+
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+--template TEMPLATE, -t TEMPLATE
+   Template to use when displaying results
+--offset OFFSET
+   .
+--pagesize PAGESIZE
+   .
+--all
+   Return all available results
+
diff --git a/docs/source/man/lesana-show.rst b/docs/source/man/lesana-show.rst
new file mode 100644
index 0000000..516f9de
--- /dev/null
+++ b/docs/source/man/lesana-show.rst
@@ -0,0 +1,45 @@
+===========
+lesana-show
+===========
+
+-------------------
+show a lesana entry
+-------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana show [--help] [--collection COLLECTION] [--template TEMPLATE] <entry>
+
+DESCRIPTION
+===========
+
+``lesana show`` will print an entry (specified by id or partial id) to
+stdout.
+
+A template can be specified with ``--template <template>`` to pretty
+print entries.
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+--collection COLLECTION, -c COLLECTION
+   The collection to work on. Default is ``.``
+--template TEMPLATE, -t TEMPLATE
+   Use the specified template to display results.
+
+TEMPLATES
+=========
+
+The templates used by ``lesana show`` are jinja2 templates.
+
+The entry fields are available as variables, and the full entry is
+available as the variable ``entry`` and can be used to give access to
+fields with names that aren't valid jinja2 variables e.g. as
+``entry.data[<field-name>]``.
+
diff --git a/docs/source/man/lesana.rst b/docs/source/man/lesana.rst
new file mode 100644
index 0000000..0d60e4f
--- /dev/null
+++ b/docs/source/man/lesana.rst
@@ -0,0 +1,76 @@
+======
+lesana
+======
+
+------------------------------
+manages collection inventories
+------------------------------
+
+:Author: valhalla@trueelena.org
+:Manual section: 1
+
+SYNOPSIS
+========
+
+lesana [--help] <command>
+
+DESCRIPTION
+===========
+
+lesana is a tool to organize collections of various kinds.  It is
+designed to have a data storage / serialization format that is friendly
+to git and other VCSs, but decent performances.
+
+To reach this aim it uses yaml_ as its serialization format, which is
+easy to store in a VCS, share between people and syncronize between
+different computers, but it also keeps an index of this data in a local
+xapian_ database in order to allow for fast searches.
+
+.. _yaml: http://yaml.org/
+.. _xapian: https://xapian.org/
+
+lesana supports collections of any kind, as long as their entries can be
+described with a mostly flat dictionary of fields of the types described
+in the documentation file ``field_types``.
+
+Some example collection schemas are provided, but one big strenght of
+lesana is the ability to customize your collection with custom fields
+either by simply writing a personalized ``settings.yaml``.
+
+OPTIONS
+=======
+
+-h, --help
+   Prints an help message and exits.
+
+COMMANDS
+========
+
+new(1)
+   Creates a new entry.
+edit(1)
+   Edits an existing entry.
+show(1)
+   Shows an existing entry.
+index(1)
+   Index some entries in the xapian cache.
+search(1)
+   Searches for entries in the xapian cache.
+export(1)
+   Exports entries from one lesana collection to another
+init(1)
+   Initialize a new lesana collection
+rm(1)
+   Removes an entry.
+
+TEXT EDITOR
+===========
+
+Many lesana subcommands will try to open files in a text editor choosen
+as follows:
+
+* first, the value of $EDITOR is tried
+* then the command ``sensible-editor``, as available under e.g. Debian
+  and its derivatives
+* lastly, it will try to fallback to ``vi``, which should be available
+  under any posix system.
diff --git a/docs/source/moving_data_between_collections.rst b/docs/source/moving_data_between_collections.rst
new file mode 100644
index 0000000..23c693e
--- /dev/null
+++ b/docs/source/moving_data_between_collections.rst
@@ -0,0 +1,41 @@
+*******************************
+Moving Data between Collections
+*******************************
+
+Entries can be exported from a lesana collection to another using the
+``lesana export`` command and a jinja2 template.
+
+The template should generate a yaml file that is a valid lesana entry
+for the destination collection and it can use the fields of the starting
+collection as variables. The variable ``entry`` is also available and
+gives complete access to the entry of the original collection, so fields
+with names that aren't valid jinja templates can be accessed as
+``entry.data[<field-name>]``.
+
+E.g. to convert between a collection with fields ``name``,
+``short-desc``, ``desc`` to a collection with fields ``name``,
+``description`` one could use the following template::
+
+   name: {{ name }}
+   description: |
+       {{ entry.data.[short-desc] }}
+
+       {{ desc | indent(width=4, first=False) }}
+
+From the origin collection you can then run the command::
+
+   lesana export <path/to/destination/collection> <path/to/template>
+
+to export all entries.
+
+You can also export just a subset of entries by adding a xapian query
+with the parameter ``--query``; you can test the search using::
+
+   lesana search --all <some search terms>
+
+and then actually run the export with::
+
+   lesana search --query '<some search terms>' <destination> <template>
+
+note that in this second command the spaces in the search query have to
+be protected from the shell.
diff --git a/docs/source/promises.rst b/docs/source/promises.rst
new file mode 100644
index 0000000..6a3b04e
--- /dev/null
+++ b/docs/source/promises.rst
@@ -0,0 +1,36 @@
+********
+Promises
+********
+
+Semantic versioning
+===================
+
+This project uses semver_.
+
+.. _semver: http://semver.org/
+
+Collection format stability
+===========================
+
+Future versions of lesana will be able to read collections written by
+older versions.
+
+Older versions in the same mayor release will also be able to work
+concurrently on the same repository.
+
+If in the future a change of formats will be required, conversions
+scripts will be written in a way that will make them as stable as
+possibile, and will have enought test data to keep them maintained for
+the time being.
+
+Disposable cache
+================
+
+Contrary to the yaml files, the xapian cache is considered disposable:
+from time to time there may be a need to delete the cache and reindex
+everything, either because of an upgrade or to perform repository
+mainteinance.
+
+Of course, effort will be made to reduce the need for this so that it
+only happens sporadically, but it will probably never completely
+disappear.
diff --git a/docs/source/reference/lesana.collection.rst b/docs/source/reference/lesana.collection.rst
new file mode 100644
index 0000000..e00613c
--- /dev/null
+++ b/docs/source/reference/lesana.collection.rst
@@ -0,0 +1,7 @@
+lesana.collection module
+========================
+
+.. automodule:: lesana.collection
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/reference/lesana.command.rst b/docs/source/reference/lesana.command.rst
new file mode 100644
index 0000000..729ec1d
--- /dev/null
+++ b/docs/source/reference/lesana.command.rst
@@ -0,0 +1,7 @@
+lesana.command module
+=====================
+
+.. automodule:: lesana.command
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/reference/lesana.data.rst b/docs/source/reference/lesana.data.rst
new file mode 100644
index 0000000..fdbd552
--- /dev/null
+++ b/docs/source/reference/lesana.data.rst
@@ -0,0 +1,7 @@
+lesana.data package
+===================
+
+.. automodule:: lesana.data
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/docs/source/reference/lesana.rst b/docs/source/reference/lesana.rst
new file mode 100644
index 0000000..bab9655
--- /dev/null
+++ b/docs/source/reference/lesana.rst
@@ -0,0 +1,24 @@
+lesana package
+==============
+
+.. automodule:: lesana
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   lesana.data
+
+Submodules
+----------
+
+.. toctree::
+   :maxdepth: 4
+
+   lesana.collection
+   lesana.command
diff --git a/docs/source/reference/modules.rst b/docs/source/reference/modules.rst
new file mode 100644
index 0000000..1ce51bd
--- /dev/null
+++ b/docs/source/reference/modules.rst
@@ -0,0 +1,7 @@
+lesana reference
+================
+
+.. toctree::
+   :maxdepth: 4
+
+   lesana