# Copyright (C) 2017-2022 J.F.Dockes
#   This program is free software; you can redistribute it and/or modify
#   it under the terms of the GNU Lesser General Public License as published by
#   the Free Software Foundation; either version 2.1 of the License, or
#   (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU Lesser General Public License for more details.
#
#   You should have received a copy of the GNU Lesser General Public License
#   along with this program; if not, write to the
#   Free Software Foundation, Inc.,
#   51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#

import os
import time
import re
from recoll import recoll

from uprclutils import audiomtypes, docfolder, uplog
import uprclutils
import uprclinit

# Tags for which we may create auxiliary tag tables for facet descent.
#
# The dict key is the tag name, used as container title in the tags tree,
# except if an a different display value was set by the user
# (_tagdisplaytag). The value is the auxiliary table name, used also
# as base for unique id and join columns (with _id) appended, and is
# also currently the recoll field name (with a provision to differ if
# needed, thanks to the currently empty _coltorclfield dict).
#
# The keys must stay same as what Minim uses as we filter them with indexTags from its
# configuration.
#
# Note: artist is actually doc["albumartist"] if set, else doc["artist"].
#
# This is the base set, with some custom translations (e.g. Group->cgroup). If a minimserver
# configuration is used, Some entries may be filtered out and some may be added.
_alltagtotable = {
    "AlbumArtist": "albumartist",
    "All Artists": "allartists",
    "Artist": "artist",  # Special processing in recolltosql: "albumartist" if set else "artist"
    "Comment": "comment",
    "Composer": "composer",
    "Conductor": "conductor",
    "Date": "date",
    "Genre": "genre",
    "Group": "cgroup",  # can't have a table named group
    "Label": "label",
    "Lyricist": "lyricist",
    "Orchestra": "orchestra",
    "Performer": "performer",
}

# Translations used when fetching fields from the recoll record. Most have the same name as the
# column.
_coltorclfield = {"allartists": "artist"}

# The actual list of tags (e.g. "Genre", "Composer") which will be shown in the tree, after reading
# the local configuration.
g_indextags = []
# The display string for the above tags. Non-identity if there are entries like Composer:Compositeur
# in the indexTags minim config
g_tagdisplaytag = {}
# Same as _alltagtotable, filtered by g_indextags
g_tagtotable = {}


def getIndexTags():
    return g_indextags


def getTagDisplayTag():
    return g_tagdisplaytag


def getTagToTable():
    return g_tagtotable


# Name of the join column for one of the tag tables. Actually, now that we use separate junction
# tables instead of columns inside the tracks table, the column name could be constant. Kept it.
def _clid(table):
    return table + "_id"


# Name of the junction table for a given tag table. Ex for genres.
# rcldocs is not an sqlite table, it's the recoll document list.
#         docidx        docidx                genre_id
# rcldocs<------>tracks<------>tracks_genres<---------->genre
def _junctb(table):
    return "tracks_" + table + "s"


# Create an empty db.
#
# There is one table for each tag (Artist, Genre, Date, etc.), listing all possible values for the
# particular tag. The table name is the value from the _alltagtotable array, <tagname> and it has 2
# columns: <tagname>_id and value.
#
# The tracks table is the "main" table, and has a record for each track, with a title column, and an
# album join column (album_id) (because a track can only belong to one album). The unique index into
# this table is the document index in the recoll document list (docidx).
#
# Because a track can have multiple values for a given tag, we use junction tables. These are named
# tracks_<tagname>s (e.g. tracks_genres). The records in these have two columns: docidx (track id)
# and and <tagname>_id (unique id from the <tagname> table)
#
# The Albums table is special because it is built according to, and stores, the file system location
# (the album title is not enough to group tracks, there could be many albums with the same
# title). Also we do tricks with grouping discs into albums etc.
#
# Note: we create all tables even if not all tags are actually used.
def _createsqdb(conn):
    c = conn.cursor()

    # Create the albums table
    try:
        c.execute("""DROP TABLE albums""")
    except:
        pass
    c.execute(
        "CREATE TABLE albums (album_id INTEGER PRIMARY KEY,"
        # artist_id is for an album_artist, either explicit from an albumartist attribute or
        # implicit if all tracks have the same artist attribute
        "artist_id INT,"
        "albtitle TEXT, albfolder TEXT, albdate TEXT, albarturi TEXT,"
        # During creation, albalb is non null only the elements of a merged album, and is the
        # album_id id for the parent. It is set by the album merging pass. When the album merge is
        # done, albalb is set to album_id for all remaining (single-disc, non merged) albums.
        "albalb INT,"
        # albtdisc is set from the tracks DISCNUMBER tag or equivalent, while initially walking the
        # docs. It signals candidates for merging. album/discs with albtdisc set are not selected
        # for display in album lists, as they are supposedly represented by their parent album. The
        # field is reset to null for albums with a discnumber which do not end up being merged.
        "albtdisc INT,"
        # Temp column while creating the table for deciding if we have a uniform artist or not. This
        # is a text version of all the artistid sets for each album track. If we were really
        # concerned about space usage, we could use a parallel temp table and drop it when done...
        "artists TEXT"
        ")"
    )

    # Create the main tracks table, which has a one-to-one relationship with the recoll document
    # array
    try:
        c.execute("""DROP TABLE tracks""")
    except:
        pass
    tracksstmt = """CREATE TABLE tracks 
                     (docidx INT, album_id INT, trackno INT, title TEXT, path TEXT)"""
    c.execute(tracksstmt)

    # Create tables for tag values (e.g. all genre values, all composer values, etc.)
    for tb in _alltagtotable.values():
        _createTagTables(c, tb)


# Create the value and junction table for a given tag
def _createTagTables(cursor, tgnm):
    # Create a table for this tag values and value ids
    try:
        cursor.execute("DROP TABLE " + tgnm)
    except:
        pass
    stmt = "CREATE TABLE " + tgnm + " (" + _clid(tgnm) + " INTEGER PRIMARY KEY, value TEXT)"
    cursor.execute(stmt)
    # Create a junction table between the tracks table and the tag values table, with docidx and
    # valueid columns, allowing multiple tag values for a given track (e.g. several
    # genres). Using a table-specific name for the id column would not be necessary, it's a
    # remnant of the time where these columns were integral to the tracks table. Still, does not
    # hurt.
    try:
        cursor.execute("DROP TABLE " + _junctb(tgnm))
    except:
        pass
    stmt = f"CREATE TABLE {_junctb(tgnm)} (docidx INT, {_clid(tgnm)} INT, " \
        f"UNIQUE(docidx, {_clid(tgnm)}))"
    cursor.execute(stmt)


# Augment indextag dict and create tables for a custom field, not part of our predefined set
def _addCustomTable(conn, indextag):
    c = conn.cursor()
    tb = indextag.lower()
    _alltagtotable[indextag] = tb
    _createTagTables(c, tb)


# Peruse the configuration to decide what tags will actually show up
# in the tree and how they will be displayed.
def _prepareTags(conn):
    global g_tagdisplaytag
    global g_tagtotable
    global g_indextags

    g_indextags = []
    g_tagdisplaytag = {}
    g_tagtotable = {}

    indextagsp = uprclinit.g_minimconfig.getindextags()
    itemtags = uprclinit.g_minimconfig.getitemtags()
    if not indextagsp:
        # Our default list of index tags (showing up as containers in the tree). The second field in
        # each pair, if set, is the customized display (e.g. ("Composer", "Compositeur"))
        indextagsp = [("Artist", ""), ("Date", ""), ("Genre", ""), ("Composer", "")]

    # Compute the actual list of index tags:
    for v, d in indextagsp:
        if v.lower() == "none":
            g_indextags = []
            g_tagdisplaytag = {}
            break
        g_indextags.append(v)
        g_tagdisplaytag[v] = d if d else v
    uplog("prepareTags: g_indextags: %s g_tagdisplaytag %s" % (g_indextags, g_tagdisplaytag))

    # Compute an array of (table name, recoll field) translations for the tags we need to process,
    # as determined by the indexTags property. Most often they are identical. This also determines
    # what fields we create tables for.
    tabtorclfield = []
    for nm in g_indextags:
        if nm not in _alltagtotable:
            _addCustomTable(conn, nm)
        tb = _alltagtotable[nm]
        g_tagtotable[nm] = tb
        rclfld = _coltorclfield[tb] if tb in _coltorclfield else tb
        uplog(f"prepareTags: using rclfield [{rclfld}] for sql [{tb}]")
        tabtorclfield.append((tb, rclfld))
        # Special case "artist" because we also want "composer" and "conductor" in there.  This is
        # so that recoll doc records will be created by _artiststorecoll(), and search for artist by
        # title will find them. This is a bit dubious because, if there are Composer or Conductor
        # indexes, we are duplicating data, and the user probably does not want to find composers in
        # the artist list. The other possible approach would (tbchecked?) be to use itemtags tables
        # but this would need additional code in uprcltags.py to duplicate the direntryforartid()
        # method and the code in uprclsearch which calls it + maybe some of the code which links
        # artists to albums and tracks probably? may worth a try anyway, it would be nicer if not
        # too complicated.
        if tb == "artist":
            tabtorclfield.append((tb, "composer"))
            tabtorclfield.append((tb, "conductor"))

    for nm in itemtags:
        if nm not in _alltagtotable:
            _addCustomTable(conn, nm)
        tb = _alltagtotable[nm]
        rclfld = _coltorclfield[tb] if tb in _coltorclfield else tb
        uplog(f"prepareTags: using rclfield [{rclfld}] for sql [{tb}]")
        tabtorclfield.append((tb, rclfld))

    uplog(f"prepareTags: tabtorclfield: {tabtorclfield}")
    return tabtorclfield


# Insert new value if not existing, return rowid of new or existing row
def _auxtableinsert(conn, tb, value):
    # uplog(f"_auxtableinsert {tb} -> {value}")
    c = conn.cursor()
    stmt = f"SELECT {_clid(tb)} FROM  {tb} WHERE value = ?"
    c.execute(stmt, (value,))
    r = c.fetchone()
    if r:
        rowid = r[0]
    else:
        stmt = f"INSERT INTO {tb}(value) VALUES(?)"
        c.execute(stmt, (value,))
        rowid = c.lastrowid
    return rowid


# tracknos like n/max are now supposedly processed by rclaudio and
# should not arrive here, but let's play it safe.
def _tracknofordoc(doc):
    try:
        return int(doc["tracknumber"].split("/")[0])
    except:
        # uplog("_tracknofordoc: doc["tracknumber"] %s title %s url %s" %
        #      (doc["tracknumber"], doc["title"], doc["url"]))
        return 1


# Detect '[disc N]' or '(disc N)' or ' disc N' at the end of an album title
# Groups         1   2       3                 4                      5
_albtitdnumre = r"(.*)(\[disc ([0-9]+)\]|\(disc ([0-9]+)\)|[ ]+disc[ ]+([0-9]+))[ ]*$"
_albtitdnumexp = re.compile(_albtitdnumre, flags=re.IGNORECASE)
_folderdnumre = r"(cd|disc)[ ]*([0-9]+)(.*)"
_folderdnumexp = re.compile(_folderdnumre, flags=re.IGNORECASE)


# Create album record for track if needed (not already there).
# The albums table is special, can't use auxtableinsert()
def _maybecreatealbum(conn, doc):
    c = conn.cursor()
    folder = docfolder(doc)

    album = doc["album"]
    if not album:
        album = uprclutils.basename(folder)
        # uplog("Using %s for alb MIME %s title %s" % (album,doc["mtype"],doc["url"]))

    # uplog(f"_maybecreatealbum: album: [{album}] folder [{folder}]")

    if doc["albumartist"]:
        albartist_id = _auxtableinsert(conn, "artist", doc["albumartist"])
    else:
        albartist_id = None

    # See if there is a discnum, either explicit or from album
    # title
    discnum = None
    if doc["discnumber"]:
        try:
            discnum = int(doc["discnumber"])
            # uplog(f"discnumber {discnum} folder {folder}")
        except:
            pass

    # Look for a disc number at the end of the album title. If it's
    # there and it's the same as the discnumber attribute, we fix the
    # title, else we leave it alone (and the merge will later fail for
    # differing titles)
    m = _albtitdnumexp.search(album)
    if m:
        # uplog("Disc number found at end of album title")
        for i in (3, 4, 5):
            if m.group(i):
                adiscnum = int(m.group(i))
                # uplog(f"Match is in group {i} adiscnum {adiscnum} discnum {discnum}")
                if not discnum:
                    discnum = adiscnum
                if adiscnum == discnum:
                    album = m.group(1)
                    # uplog(f"album title changed to [{album}]")
                break

    if not discnum:
        m = _folderdnumexp.search(uprclutils.basename(folder))
        if m:
            discnum = int(m.group(2))

    # See if this albumdisc already exists (created for a previous track)
    stmt = "SELECT album_id, artist_id FROM albums WHERE albtitle = ? AND albfolder = ?"
    wcols = [album, folder]
    if discnum:
        stmt += " AND albtdisc = ?"
        wcols.append(discnum)
    # uplog(f"maybecreatealbum: {stmt} {wcols}")
    c.execute(stmt, wcols)
    r = c.fetchone()
    if r:
        # uplog("maybecreatealbum: album found")
        album_id = r[0]
    else:
        c.execute(
            """INSERT INTO 
        albums(albtitle, albfolder, artist_id, albdate, albtdisc, artists) 
        VALUES (?,?,?,?,?,?)""",
            (album, folder, albartist_id, doc["date"], discnum, ""),
        )
        album_id = c.lastrowid
        # uplog(f"New album {album_id} {album} disc {discnum} artist {albartist_id} folder {folder}")

    return album_id


# Add a track's artists set to the album auxiliary "artists" column, used in the end to determine an
# album artist if none was explicitely set.
# The contents of the column is a string made of string representations of Python sets of
# ints. E.g. "|{1234, 12345}|{99}|{23, 24}"
# At the end (all tracks seen), we reconstruct the sets and intersect them. If the intersection is
# not empty, we make a random element inside it the album artist. See _setalbumartists()
# It seems that we could accomplish the same thing by querying the db for each of the album tracks
# artists at the end ? I can't see the reason for this approach any more, but maybe there is one...
def _updatealbartistlist(conn, album_id, rowids):
    if not rowids:
        return
    c = conn.cursor()
    addstr = repr(rowids)
    stmt = f"UPDATE albums SET artists = artists || '|' || ? WHERE album_id = ?"
    values = (addstr, album_id)
    c.execute(stmt, values)


# Setting album covers needs to wait until we have scanned all tracks so that we can select
# a consistant embedded art (first track in path order), as recoll scanning is in unsorted
# directory order.
def _setalbumcovers(conn, folders):
    rcldocs = folders.rcldocs()
    c = conn.cursor()
    c.execute("""SELECT album_id,albtitle FROM albums""")
    for r in c:
        albid = r[0]
        albtitle = r[1]
        c1 = conn.cursor()
        stmt = """SELECT docidx FROM tracks WHERE album_id = ? ORDER BY path"""
        c1.execute(stmt, (albid,))
        for r1 in c1:
            docidx = r1[0]
            doc = rcldocs[docidx]
            arturi = folders.docarturi(
                doc,
                preferfolder=True,
                albtitle=albtitle,
            )
            if arturi:
                cupd = conn.cursor()
                # uplog(f"Setting albid {albid} albarturi to {arturi}")
                cupd.execute("UPDATE albums SET albarturi = ?  WHERE album_id = ?", (arturi, albid))
                break


# After the pass on tracks, look for all albums where the album artist is not set but all the tracks
# have the same artist, and set the album artist.
def _setalbumartists(conn):
    c = conn.cursor()
    # Get all albums without an albumartist where the same_track_artist flag is on
    c.execute("SELECT album_id, artists, albtitle FROM albums WHERE artist_id IS NULL")
    # For all albumartist-less albums
    for r in c:
        # Default to various artist
        albumartist = variousartistsid
        if r[1]:
            # Eval the string back to list of artist sets, and intersect to see if we have a common
            # artist
            s = r[1].lstrip("|")
            artsets = s.split("|")
            artsets = [eval(artset) for artset in artsets]
            if artsets:
                intersect = artsets[0]
                for artset in artsets[1:]:
                    intersect = intersect.intersection(artset)
                # uplog(f"_setalbumartists: INTERSECT for [{r[2]}]: {intersect}")
                if intersect:
                    # if multiple values, have to chose one...
                    albumartist = [v for v in intersect][0]
                    # uplog(f"Using albumartist {albumartist} for {r[2]}")

        if albumartist == variousartistsid:
            # uplog(f"Using albumartist Various Artists for {r[2]}")
            pass
        c1 = conn.cursor()
        c1.execute("UPDATE albums SET artist_id = ? WHERE album_id = ?", (albumartist, r[0]))


# Add albums to the recoll index so they can be searched for
def _albumstorecoll(conn, rcldb):
    c = conn.cursor()
    #                 0        1          2          3         4         5
    stmt = """SELECT album_id, albfolder, albtitle, albarturi, albdate, artist.value
      FROM albums LEFT JOIN artist ON artist.artist_id = albums.artist_id
      WHERE albtdisc is NULL"""
    c.execute(stmt, ())
    for r in c:
        udi = "albid" + str(r[0])
        doc = rcldb.doc()
        doc["title"] = r[2]
        doc["mtype"] = "inode/directory"
        if r[5]:
            doc["albumartist"] = r[5]
            doc["artist"] = r[5]
        doc["url"] = "file://" + r[1]
        # uplog(f"_albumstorecoll: indexing album {doc['title']}")
        rcldb.addOrUpdate(udi, doc)


# Add artists to the recoll index so they can be searched for
def _artiststorecoll(conn, rcldb):
    c = conn.cursor()
    #                 0         1
    stmt = "SELECT artist_id, value    FROM artist"
    c.execute(stmt, ())
    for r in c:
        udi = "artid" + str(r[0])
        doc = rcldb.doc()
        doc["title"] = r[1]
        doc["mtype"] = "inode/directory"
        doc["url"] = "file://artists/" + r[1]  # Not used ever
        # uplog(f"_artiststorecoll: indexing artist {doc['title']}")
        rcldb.addOrUpdate(udi, doc)


# Check that the numbers are sequential
def _checkseq(seq):
    num = seq[0]
    if not num:
        return False
    for e in seq[1:]:
        if e != num + 1:
            return False
        num = e
    return True


def _membertotopalbum(conn, memberalbid):
    c = conn.cursor()
    c.execute("""SELECT * FROM albums WHERE album_id = ?""", (memberalbid,))
    cols = [desc[0] for desc in c.description]
    # Get array of column values, and set primary key and albtdisc to
    # None before inserting the copy
    tdiscindex = cols.index("albtdisc")
    v = [e for e in c.fetchone()]
    v[0] = None
    v[tdiscindex] = None
    c.execute("""INSERT INTO albums VALUES (%s)""" % ",".join("?" * len(v)), v)
    return c.lastrowid


# Only keep albums in the same folder or in a sibling folder
def _mergealbumsfilterfolders(folder, rows, colidx):
    parent = uprclutils.dirname(folder)
    rows1 = []
    for row in rows:
        if row[colidx] == folder or uprclutils.dirname(row[colidx]) == parent:
            rows1.append(row)
    return rows1


## TBD: folder match filter
def _createmergedalbums(conn):
    c = conn.cursor()

    # Remember already merged
    merged = set()

    # All candidates for merging: albums with a disc number not yet
    # merged (albalb is null)
    c.execute(
        """SELECT album_id, albtitle, artist_id, albfolder FROM albums
      WHERE albalb IS NULL AND albtdisc IS NOT NULL"""
    )
    c1 = conn.cursor()
    for r in c:
        albid = r[0]
        if albid in merged:
            continue
        albtitle = r[1]
        artist = r[2]
        folder = r[3]

        # uplog(f"_createmergedalbums: albid {albid} artist_id {artist} albtitle {albtitle}")

        # Look for albums not already in a group, with the same title and artist
        if artist:
            c1.execute(
                """SELECT album_id, albtdisc, albfolder FROM albums
                WHERE albtitle = ? AND artist_id = ?
                AND albalb is NULL AND albtdisc IS NOT NULL""",
                (albtitle, artist),
            )
        else:
            c1.execute(
                """SELECT album_id, albtdisc, albfolder FROM albums
                WHERE albtitle = ? AND artist_id IS NULL
                AND albalb is NULL AND albtdisc IS NOT NULL""",
                (albtitle,),
            )

        rows = c1.fetchall()
        rows1 = _mergealbumsfilterfolders(folder, rows, 2)

        # uplog(f"_createmergedalbums: got {len(rows1)} possible(s) title {albtitle}")

        if len(rows1) > 1:
            albids = [row[0] for row in rows1]
            dnos = sorted([row[1] for row in rows1])
            if not _checkseq(dnos):
                uplog(f"mergealbums: not merging bad seq {dnos} for albtitle {albtitle}")
                c1.execute(
                    f"UPDATE albums SET albtdisc = NULL "
                    f"WHERE album_id in ({','.join('?'*len(albids))})",
                    albids,
                )
                continue

            # Create record for whole album by copying the first
            # record, setting its album_id and albtdisc to NULL
            topalbid = _membertotopalbum(conn, albids[0])

            # Update all album disc members with the top album id
            values = [
                topalbid,
            ] + albids
            c1.execute(
                f"UPDATE albums SET albalb = ? " f"WHERE album_id in ({','.join('?'*len(albids))})",
                values,
            )
            merged.update(albids)
            # uplog("_createmergedalbums: merged: %s" % albids)

        elif len(rows1) == 1:
            # Album with a single disc having a discnumber. Just unset
            # the discnumber, we won't use it and its presence would
            # prevent the album from showing up. Alternatively we
            # could set albalb = album_id?
            # uplog("Setting albtdisc to NULL albid %d" % albid)
            c1.execute("UPDATE albums SET albtdisc = NULL WHERE album_id= ?", (albid,))

    # finally, set albalb to albid for all single-disc albums
    c.execute("UPDATE albums SET albalb = album_id WHERE albtdisc IS NULL")


# Parse date: partly duplicated from rclaudio, see comments about date formats there.  The date
# field is a bit of a mess because up to some point recoll had an alias from date to dmtime, which
# resulted in no date field and a calendar date (instead of uxtime) in dmtime. So we take the date
# value from 'date' if set, else 'dmtime', and then process it here. All reasonable date formats
# begin with YYYY[-MM[-DD]] and this is the part we keep.
def parsedate(dt):
    if len(dt) > 10:
        dt = dt[0:10]
    l = dt.split("-")
    if len(l) > 3 or len(l) == 2 or len(l[0]) != 4 or l[0] == "0000":
        return None
    # uplog("parsedate: -> %s" % dt)
    return dt


# Create the db and fill it up with the values we need, taken out of
# the recoll records list
def recolltosql(conn, folders):
    rcldocs = folders.rcldocs()
    start = time.time()

    _createsqdb(conn)
    tabtorclfield = _prepareTags(conn)
    # uplog("Tagscreate: tabtorclfield: %s"%tabtorclfield)

    maxcnt = 0
    totcnt = 0
    c = conn.cursor()

    # A generic "Various Artists" tag value to be used for Albumartist if there are multiple artists
    # and no explicit AlbumArtist value. Set this as global, no need to query for it every time it's
    # needed.
    global variousartistsid
    variousartistsid = _auxtableinsert(conn, "artist", "Various Artists")

    for docidx in range(len(rcldocs)):
        doc = rcldocs[docidx]
        totcnt += 1

        if totcnt % 1000 == 0:
            time.sleep(0)

        # No need to include non-audio or non-tagged types
        if (
            doc["mtype"] not in audiomtypes
            or doc["mtype"] == "inode/directory"
            or doc["mtype"] == "audio/x-mpegurl"
        ):
            continue

        # Album creation ?
        album_id = _maybecreatealbum(conn, doc)

        trackno = _tracknofordoc(doc)

        path = uprclutils.docpath(doc)

        # Misc tag values:
        # Set base values for column names, values list, placeholders. Done this way because we used
        # to add to the lists for the tag values (which now use separate junction tables).
        columns = ["docidx", "album_id", "trackno", "title", "path"]
        values = [docidx, album_id, trackno, doc["title"], path]
        placehold = ["?", "?", "?", "?", "?"]
        for tb, rclfld in tabtorclfield:
            value = doc[rclfld]
            # Special processing for some fields
            if rclfld == "date":
                # See comment in parsedate
                if not value:
                    value = doc["dmtime"]
                if value:
                    value = parsedate(value)
            elif rclfld == "artist":
                value = doc["albumartist"]
                if not value:
                    value = doc["artist"]
            if not value:
                continue
            # rclaudio.py concatenates multiple values, using " | " as separator.
            valuelist = set(value.split(" | "))
            rowids = set()
            for value in valuelist:
                # Possibly insert in appropriate table (if value not already there), and
                # insert record in junction table for the corresponding field.
                rowid = _auxtableinsert(conn, tb, value)
                rowids.add(rowid)
                stmt = f"INSERT OR IGNORE INTO {_junctb(tb)}(docidx, {_clid(tb)}) VALUES (?, ?)"
                jvals = [docidx, rowid]
                # uplog(f"EXECUTING {stmt} values {jvals}")
                c.execute(stmt, jvals)
            if tb == "artist" and rowids:
                _updatealbartistlist(conn, album_id, rowids)
        # Create the main record in the tracks table.
        stmt = "INSERT INTO tracks(" + ",".join(columns) + ") VALUES(" + ",".join(placehold) + ")"
        c.execute(stmt, values)
    
    ## End Big doc loop

    #t1 = time.time()
    #uplog(f"recolltosql: docwalk: {t1-start:.1f} Seconds")
    _setalbumartists(conn)
    _setalbumcovers(conn, folders)
    #t2 = time.time()
    #uplog(f"recolltosql: setalbumxx: {t2-t1:.1f} Seconds")
    _createmergedalbums(conn)
    #t1 = time.time()
    #uplog(f"recolltosql: createmergedalbums: {t1-t2:.1f} Seconds")
    conn.commit()
    #t2 = time.time()
    #uplog(f"recolltosql: commit: {t2-t1:.1f} Seconds")
    rcldb = recoll.connect(confdir=uprclinit.getRclConfdir(), writable=True)
    _albumstorecoll(conn, rcldb)
    _artiststorecoll(conn, rcldb)
    end = time.time()
    uplog(f"recolltosql: processed {totcnt} docs in {end-start:.1f} Seconds")
