From 2a824fab4e4b48184583b639f6753ab051f4dd55 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sat, 25 Mar 2017 18:37:36 +0100 Subject: Start writing a tellico importer --- scripts/tellico2lesana | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 scripts/tellico2lesana diff --git a/scripts/tellico2lesana b/scripts/tellico2lesana new file mode 100755 index 0000000..8de8bf9 --- /dev/null +++ b/scripts/tellico2lesana @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +from xml.etree import ElementTree +import zipfile + +import guacamole + + +XML_NS = '{http://periapsis.org/tellico/}' + +# https://docs.kde.org/trunk5/en/extragear-office/tellico/field-type-values.html +F_TYPE_MAP = { + '1': 'string', + '2': 'text', + '3': 'string', + '4': 'bool', + '6': 'integer', + '7': 'url', + '8': 'list', # single column table + '10': 'file', + '12': 'timestamp', # date + '14': 'integer', # rating + + } + + +class T2L(guacamole.Command): + """ + Manage collections + """ + + def register_arguments(self, parser): + parser.add_argument( + 'file', + help='Tellico file to convert to lesana.' + ) + + def invoked(self, ctx): + with zipfile.ZipFile(ctx.args.file, 'r') as zp: + tree = ElementTree.parse(zp.open('tellico.xml')) + # open collection + xml_collection = tree.getroot().find(XML_NS+'collection') + + # get collection settings + title = xml_collection.attrib['title'] + xml_fields = xml_collection.find(XML_NS+'fields') + fields = [] + for xf in xml_fields: + print(xf.attrib) + f_type = F_TYPE_MAP.get(xf.attrib['format']) + # TODO: support fields with the multiple values flag + # (they should probably become lists) + fields.append({ + 'name': xf.attrib['title'], + 'type': f_type, + }) + # TODO: create a collection with the following settings: + import ruamel.yaml + print(ruamel.yaml.dump({'title': title, 'fields': fields})) + + # import data + + + +if __name__ == '__main__': + T2L().main() -- cgit v1.2.3 From 573411b3d1248fb1bb8b58f4143dc223aea2cc85 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sat, 22 Apr 2017 19:18:41 +0200 Subject: Pass settings to Collection.init --- lesana/collection.py | 10 +++++++++- tests/test_collection.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lesana/collection.py b/lesana/collection.py index 1f488bc..5b80d3b 100644 --- a/lesana/collection.py +++ b/lesana/collection.py @@ -304,7 +304,13 @@ class Collection(object): cache.close() @classmethod - def init(cls, directory=None, git_enabled=True, edit_file=None): + def init( + cls, + directory=None, + git_enabled=True, + edit_file=None, + settings={} + ): """ Initialize a lesana repository @@ -316,6 +322,7 @@ class Collection(object): own errors. """ c_dir = os.path.abspath(directory or '.') + os.makedirs(c_dir, exist_ok=True) if git_enabled: # Try to initalize a git repo if git_available: @@ -351,6 +358,7 @@ class Collection(object): ).decode('utf-8') skel_dict = ruamel.yaml.load(skel, ruamel.yaml.RoundTripLoader) skel_dict['git'] = git_enabled + skel_dict.update(settings) with open(filepath, 'w') as fp: ruamel.yaml.dump( skel_dict, diff --git a/tests/test_collection.py b/tests/test_collection.py index e17e044..bcd45e7 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -298,6 +298,25 @@ class testCollectionCreation(unittest.TestCase): self.assertTrue(self._find_file_in_git_index(fname, repo.index)) shutil.rmtree(tmpdir) + def test_init_custom_settings(self): + tmpdir = tempfile.mkdtemp() + collection = lesana.Collection.init( + tmpdir, + edit_file=self.do_nothing, + settings={ + 'name': 'A different name', + 'fields': [ + {'name': 'title', 'type': 'string'}, + {'name': 'author', 'type': 'string'}, + ], + }, + ) + self.assertIsInstance(collection, lesana.Collection) + self.assertTrue(os.path.isfile(os.path.join(tmpdir, 'settings.yaml'))) + self.assertEqual(collection.settings['name'], 'A different name') + self.assertEqual(len(collection.settings['fields']), 2) + shutil.rmtree(tmpdir) + if __name__ == '__main__': unittest.main() -- cgit v1.2.3 From fd0b9834b09d4afe5bdaccc42e3e5c73d64d25d6 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sat, 22 Apr 2017 19:24:42 +0200 Subject: tellico2lesana: create a lesana collection --- scripts/tellico2lesana | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/scripts/tellico2lesana b/scripts/tellico2lesana index 8de8bf9..e820069 100755 --- a/scripts/tellico2lesana +++ b/scripts/tellico2lesana @@ -3,6 +3,8 @@ from xml.etree import ElementTree import zipfile +import lesana + import guacamole @@ -30,9 +32,14 @@ class T2L(guacamole.Command): """ def register_arguments(self, parser): + parser.add_argument( + '-c', '--collection', + help='Name of the new lesana collection', + default=None, + ) parser.add_argument( 'file', - help='Tellico file to convert to lesana.' + help='Tellico file to convert to lesana.', ) def invoked(self, ctx): @@ -54,9 +61,19 @@ class T2L(guacamole.Command): 'name': xf.attrib['title'], 'type': f_type, }) - # TODO: create a collection with the following settings: - import ruamel.yaml - print(ruamel.yaml.dump({'title': title, 'fields': fields})) + # Create a collection with the settings we have loaded + directory = ctx.args.collection or ctx.args.file.replace( + '.tc', + '.lesana' + ) + lesana.collection.Collection.init( + directory=directory, + git_enabled=False, + settings={ + 'name': title, + 'fields': fields, + } + ) # import data -- cgit v1.2.3 From bd5a61a44e6925d2003547cf9fbfaf1066c65aa1 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sat, 20 May 2017 11:30:46 +0200 Subject: additional value for tellico field types --- scripts/tellico2lesana | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tellico2lesana b/scripts/tellico2lesana index e820069..c3cbeef 100755 --- a/scripts/tellico2lesana +++ b/scripts/tellico2lesana @@ -12,6 +12,7 @@ XML_NS = '{http://periapsis.org/tellico/}' # https://docs.kde.org/trunk5/en/extragear-office/tellico/field-type-values.html F_TYPE_MAP = { + '0': 'string', # not in the specs, but seen in the wild '1': 'string', '2': 'text', '3': 'string', -- cgit v1.2.3 From 95f32e6c767a03bc1b16bcd5a596b77cb49de9e4 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sat, 20 May 2017 11:50:20 +0200 Subject: Start reading entries --- scripts/tellico2lesana | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/tellico2lesana b/scripts/tellico2lesana index c3cbeef..1c579c9 100755 --- a/scripts/tellico2lesana +++ b/scripts/tellico2lesana @@ -59,8 +59,9 @@ class T2L(guacamole.Command): # TODO: support fields with the multiple values flag # (they should probably become lists) fields.append({ - 'name': xf.attrib['title'], + 'name': xf.attrib['name'], 'type': f_type, + 'help': xf.attrib['title'], }) # Create a collection with the settings we have loaded directory = ctx.args.collection or ctx.args.file.replace( @@ -77,6 +78,12 @@ class T2L(guacamole.Command): ) # import data + for xe in xml_collection.findall(XML_NS+'entry'): + data = { + 'uid': xe.attrib['id'] + } + for xfield in xe.getchildren(): + print(xfield, xfield.text) -- cgit v1.2.3 From 4fe7b851d6ffc86d70577407774532921403ac8d Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sun, 21 May 2017 13:01:23 +0200 Subject: Fix creation of an entry with uid but no filename --- lesana/collection.py | 3 ++- tests/test_collection.py | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lesana/collection.py b/lesana/collection.py index 5b80d3b..d3715d1 100644 --- a/lesana/collection.py +++ b/lesana/collection.py @@ -26,7 +26,8 @@ class Entry(object): self.uid, ext = os.path.splitext(os.path.basename(self.fname)) else: self.uid = uuid.uuid4().hex - self.fname = self.uid + '.yaml' + if not self.fname: + self.fname = self.uid + '.yaml' def __str__(self): label = self.collection.settings.get('entry_label', None) diff --git a/tests/test_collection.py b/tests/test_collection.py index bcd45e7..891a439 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -164,6 +164,30 @@ class testEntries(unittest.TestCase): '{uid}: {name}'.format(uid=uid, name='Another item') ) + def test_entry_creation_uid_but_no_filename(self): + fname = '11189ee47ddf4796b718a483b379f976.yaml' + with open(os.path.join(self.basepath, fname)) as fp: + data = ruamel.yaml.safe_load(fp) + data['uid'] = '11189ee47ddf4796b718a483b379f976' + entry = lesana.Entry(self.collection, data=data) + self.assertEqual(entry.fname, fname) + + def test_entry_creation_no_uid_no_filename(self): + fname = '11189ee47ddf4796b718a483b379f976.yaml' + with open(os.path.join(self.basepath, fname)) as fp: + data = ruamel.yaml.safe_load(fp) + entry = lesana.Entry(self.collection, data=data) + self.assertIsNotNone(entry.uid) + self.assertIsNotNone(entry.fname) + + def test_entry_creation_filename_but_no_uid(self): + fname = '11189ee47ddf4796b718a483b379f976.yaml' + uid = '11189ee47ddf4796b718a483b379f976' + with open(os.path.join(self.basepath, fname)) as fp: + data = ruamel.yaml.safe_load(fp) + entry = lesana.Entry(self.collection, data=data, fname=fname) + self.assertEqual(entry.uid, uid) + class testComplexCollection(unittest.TestCase): @classmethod -- cgit v1.2.3 From 15ae1a931e51f96b644c7003d6420bbddffbc2e1 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sun, 21 May 2017 13:13:43 +0200 Subject: tellico2lesana: start creating entries --- scripts/tellico2lesana | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/scripts/tellico2lesana b/scripts/tellico2lesana index 1c579c9..7b4df16 100755 --- a/scripts/tellico2lesana +++ b/scripts/tellico2lesana @@ -8,7 +8,7 @@ import lesana import guacamole -XML_NS = '{http://periapsis.org/tellico/}' +NS = {'tellico': 'http://periapsis.org/tellico/'} # https://docs.kde.org/trunk5/en/extragear-office/tellico/field-type-values.html F_TYPE_MAP = { @@ -43,32 +43,53 @@ class T2L(guacamole.Command): help='Tellico file to convert to lesana.', ) + def read_field_data(self, xfield): + if xfield.iter().__next__(): + data = [] + for child in xfield: + data.append(self.read_field_data(child)) + else: + data = xfield.text + return data + + def invoked(self, ctx): with zipfile.ZipFile(ctx.args.file, 'r') as zp: tree = ElementTree.parse(zp.open('tellico.xml')) # open collection - xml_collection = tree.getroot().find(XML_NS+'collection') + xml_collection = tree.getroot().find('tellico:collection', NS) # get collection settings title = xml_collection.attrib['title'] - xml_fields = xml_collection.find(XML_NS+'fields') + xml_fields = xml_collection.find('tellico:fields', NS) fields = [] for xf in xml_fields: - print(xf.attrib) f_type = F_TYPE_MAP.get(xf.attrib['format']) # TODO: support fields with the multiple values flag # (they should probably become lists) - fields.append({ + try: + flags = int(xf.attrib['flags']) + except ValueError: + flags = 0 + if flags % 2 == 1: + l_type = f_type + f_type = 'list' + else: + l_type = None + field = { 'name': xf.attrib['name'], 'type': f_type, 'help': xf.attrib['title'], - }) + } + if l_type: + field['list'] = l_type + fields.append(field) # Create a collection with the settings we have loaded directory = ctx.args.collection or ctx.args.file.replace( '.tc', '.lesana' ) - lesana.collection.Collection.init( + self.collection = lesana.collection.Collection.init( directory=directory, git_enabled=False, settings={ @@ -78,12 +99,19 @@ class T2L(guacamole.Command): ) # import data - for xe in xml_collection.findall(XML_NS+'entry'): + for xe in xml_collection.findall('tellico:entry', NS): data = { 'uid': xe.attrib['id'] } for xfield in xe.getchildren(): - print(xfield, xfield.text) + field_name = xfield.tag.replace( + '{'+NS['tellico']+'}', + '') + data[field_name] = self.read_field_data(xfield) + new_entry = lesana.collection.Entry(self.collection, data=data) + self.collection.save_entries([new_entry]) + self.collection.update_cache() + -- cgit v1.2.3 From 043e99bf5416e0899c02490fa600be4cfd049da1 Mon Sep 17 00:00:00 2001 From: Elena ``of Valhalla'' Grandi Date: Sat, 5 Aug 2017 18:41:38 +0200 Subject: Correctly load date fields --- scripts/tellico2lesana | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/scripts/tellico2lesana b/scripts/tellico2lesana index 7b4df16..0b0ec64 100755 --- a/scripts/tellico2lesana +++ b/scripts/tellico2lesana @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import datetime from xml.etree import ElementTree import zipfile @@ -44,7 +45,23 @@ class T2L(guacamole.Command): ) def read_field_data(self, xfield): - if xfield.iter().__next__(): + if xfield.tag in self.date_fields: + for child in xfield: + if 'year' in child.tag: + year = child.text + elif 'month' in child.tag: + month = child.text + elif 'day' in child.tag: + day = child.text + try: + data = datetime.date( + int(year), + int(month), + int(day) + ) + except ValueError: + data = None + elif xfield.iter().__next__(): data = [] for child in xfield: data.append(self.read_field_data(child)) @@ -62,9 +79,14 @@ class T2L(guacamole.Command): # get collection settings title = xml_collection.attrib['title'] xml_fields = xml_collection.find('tellico:fields', NS) + self.date_fields = [] fields = [] for xf in xml_fields: - f_type = F_TYPE_MAP.get(xf.attrib['format']) + if xf.attrib['type'] == '12': + self.date_fields.append( + '{' + NS['tellico'] + '}' + xf.attrib['name'] + ) + f_type = F_TYPE_MAP.get(xf.attrib['type']) # TODO: support fields with the multiple values flag # (they should probably become lists) try: -- cgit v1.2.3