# This file is part of Buildbot.  Buildbot is free software: you can
# redistribute it and/or modify it under the terms of the GNU General Public
# License as published by the Free Software Foundation, version 2.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 51
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Copyright Buildbot Team Members

import cPickle
import textwrap
import os
import sys

from twisted.persisted import styles

from buildbot.db import util
from buildbot.db.schema import base
from buildbot.util import json

# This is version 1, so it introduces a lot of new tables over version 0,
# which had no database.

TABLES = [
    # the schema here is defined as version 1
    textwrap.dedent("""
        CREATE TABLE version (
            version INTEGER NOT NULL -- contains one row, currently set to 1
        );
    """),

    # last_access is used for logging, to record the last time that each
    # client (or rather class of clients) touched the DB. The idea is that if
    # something gets weird, you can check this and discover that you have an
    # older tool (which uses a different schema) mucking things up.
    textwrap.dedent("""
        CREATE TABLE last_access (
            `who` VARCHAR(256) NOT NULL, -- like 'buildbot-0.8.0'
            `writing` INTEGER NOT NULL, -- 1 if you are writing, 0 if you are reading
            -- PRIMARY KEY (who, writing),
            `last_access` TIMESTAMP     -- seconds since epoch
        );
    """),

    textwrap.dedent("""
        CREATE TABLE changes_nextid (next_changeid INTEGER);
    """),

    textwrap.dedent("""
        -- Changes are immutable: once added, never changed
        CREATE TABLE changes (
            `changeid` INTEGER PRIMARY KEY NOT NULL, -- also serves as 'change number'
            `author` VARCHAR(1024) NOT NULL,
            `comments` VARCHAR(1024) NOT NULL, -- too short?
            `is_dir` SMALLINT NOT NULL, -- old, for CVS
            `branch` VARCHAR(1024) NULL,
            `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs?
            `revlink` VARCHAR(256) NULL,
            `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change
            `category` VARCHAR(256) NULL
        );
    """),

    textwrap.dedent("""
        CREATE TABLE change_links (
            `changeid` INTEGER NOT NULL,
            `link` VARCHAR(1024) NOT NULL
        );
    """),

    textwrap.dedent("""
        CREATE TABLE change_files (
            `changeid` INTEGER NOT NULL,
            `filename` VARCHAR(1024) NOT NULL
        );
    """),

    textwrap.dedent("""
        CREATE TABLE change_properties (
            `changeid` INTEGER NOT NULL,
            `property_name` VARCHAR(256) NOT NULL,
            `property_value` VARCHAR(1024) NOT NULL -- too short?
        );
    """),

    # Scheduler tables
    textwrap.dedent("""
        CREATE TABLE schedulers (
            `schedulerid` INTEGER PRIMARY KEY, -- joins to other tables
            `name` VARCHAR(127) UNIQUE NOT NULL,
            `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary
        );
    """),

    textwrap.dedent("""
        CREATE TABLE scheduler_changes (
            `schedulerid` INTEGER,
            `changeid` INTEGER,
            `important` SMALLINT
        );
    """),

    textwrap.dedent("""
        CREATE TABLE scheduler_upstream_buildsets (
            `buildsetid` INTEGER,
            `schedulerid` INTEGER,
            `active` SMALLINT
        );
    """),

    # SourceStamps
    textwrap.dedent("""
        -- SourceStamps are immutable: once added, never changed
        CREATE TABLE sourcestamps (
            `id` INTEGER PRIMARY KEY,
            `branch` VARCHAR(256) default NULL,
            `revision` VARCHAR(256) default NULL,
            `patchid` INTEGER default NULL
        );
    """),
    textwrap.dedent("""
        CREATE TABLE patches (
            `id` INTEGER PRIMARY KEY,
            `patchlevel` INTEGER NOT NULL,
            `patch_base64` TEXT NOT NULL, -- encoded bytestring
            `subdir` TEXT -- usually NULL
        );
    """),
    textwrap.dedent("""
        CREATE TABLE sourcestamp_changes (
            `sourcestampid` INTEGER NOT NULL,
            `changeid` INTEGER NOT NULL
        );
    """),

    # BuildRequests
    textwrap.dedent("""
        -- BuildSets are mutable. Python code may not cache them. Every
        -- BuildRequest must have exactly one associated BuildSet.
        CREATE TABLE buildsets (
            `id` INTEGER PRIMARY KEY NOT NULL,
            `external_idstring` VARCHAR(256),
            `reason` VARCHAR(256),
            `sourcestampid` INTEGER NOT NULL,
            `submitted_at` INTEGER NOT NULL,
            `complete` SMALLINT NOT NULL default 0,
            `complete_at` INTEGER,
            `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py
             -- results is NULL until complete==1
        );
    """),
    textwrap.dedent("""
        CREATE TABLE buildset_properties (
            `buildsetid` INTEGER NOT NULL,
            `property_name` VARCHAR(256) NOT NULL,
            `property_value` VARCHAR(1024) NOT NULL -- too short?
        );
    """),

    textwrap.dedent("""
        -- the buildrequests table represents the queue of builds that need to be
        -- done. In an idle buildbot, all requests will have complete=1.
        -- BuildRequests are mutable. Python code may not cache them.
        CREATE TABLE buildrequests (
            `id` INTEGER PRIMARY KEY NOT NULL,

            -- every BuildRequest has a BuildSet
            -- the sourcestampid and reason live in the BuildSet
            `buildsetid` INTEGER NOT NULL,

            `buildername` VARCHAR(256) NOT NULL,

            `priority` INTEGER NOT NULL default 0,

            -- claimed_at is the time at which a master most recently asserted that
            -- it is responsible for running the build: this will be updated
            -- periodically to maintain the claim
            `claimed_at` INTEGER default 0,

            -- claimed_by indicates which buildmaster has claimed this request. The
            -- 'name' contains hostname/basedir, and will be the same for subsequent
            -- runs of any given buildmaster. The 'incarnation' contains bootime/pid,
            -- and will be different for subsequent runs. This allows each buildmaster
            -- to distinguish their current claims, their old claims, and the claims
            -- of other buildmasters, to treat them each appropriately.
            `claimed_by_name` VARCHAR(256) default NULL,
            `claimed_by_incarnation` VARCHAR(256) default NULL,

            `complete` INTEGER default 0, -- complete=0 means 'pending'

             -- results is only valid when complete==1
            `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py

            `submitted_at` INTEGER NOT NULL,

            `complete_at` INTEGER
        );
    """),

    textwrap.dedent("""
        -- this records which builds have been started for each request
        CREATE TABLE builds (
            `id` INTEGER PRIMARY KEY NOT NULL,
            `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number)
            -- 'number' is scoped to both the local buildmaster and the buildername
            `brid` INTEGER NOT NULL, -- matches buildrequests.id
            `start_time` INTEGER NOT NULL,
            `finish_time` INTEGER
        );
    """),
]

class Upgrader(base.Upgrader):
    def upgrade(self):
        self.test_unicode()
        self.add_tables()
        self.migrate_changes()
        self.set_version()

    def test_unicode(self):
        # first, create a test table
        c = self.conn.cursor()
        c.execute("CREATE TABLE test_unicode (`name` VARCHAR(100))")
        q = util.sql_insert(self.dbapi, 'test_unicode', ["name"])
        try:
            val = u"Frosty the \N{SNOWMAN}"
            c.execute(q, [val])
            c.execute("SELECT * FROM test_unicode")
            row = c.fetchall()[0]
            if row[0] != val:
                raise UnicodeError("Your database doesn't support unicode data; for MySQL, set the default collation to utf8_general_ci.")
        finally:
            pass
            c.execute("DROP TABLE test_unicode")

    def add_tables(self):
        # first, add all of the tables
        c = self.conn.cursor()
        for t in TABLES:
            try:
                c.execute(t)
            except:
                print >>sys.stderr, "error executing SQL query: %s" % t
                raise

    def _addChangeToDatabase(self, change, cursor):
        # strip None from any of these values, just in case
        def remove_none(x):
            if x is None: return u""
            elif isinstance(x, str):
                return x.decode("utf8")
            else:
                return x
        try:
            values = tuple(remove_none(x) for x in
                             (change.number, change.who,
                              change.comments, change.isdir,
                              change.branch, change.revision, change.revlink,
                              change.when, change.category))
        except UnicodeDecodeError, e:
            raise UnicodeError("Trying to import change data as UTF-8 failed.  Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e))

        q = util.sql_insert(self.dbapi, 'changes',
            """changeid author comments is_dir branch revision
               revlink when_timestamp category""".split())
        cursor.execute(q, values)

        for link in change.links:
            cursor.execute(util.sql_insert(self.dbapi, 'change_links', ('changeid', 'link')),
                          (change.number, link))

        # sometimes change.files contains nested lists -- why, I do not know!  But we deal with
        # it all the same - see bug #915. We'll assume for now that change.files contains *either*
        # lists of filenames or plain filenames, not both.
        def flatten(l):
            if l and type(l[0]) == list:
                rv = []
                for e in l:
                    if type(e) == list:
                        rv.extend(e)
                    else:
                        rv.append(e)
                return rv
            else:
                return l
        for filename in flatten(change.files):
            cursor.execute(util.sql_insert(self.dbapi, 'change_files', ('changeid', 'filename')),
                          (change.number, filename))
        for propname,propvalue in change.properties.properties.items():
            encoded_value = json.dumps(propvalue)
            cursor.execute(util.sql_insert(self.dbapi, 'change_properties',
                                  ('changeid', 'property_name', 'property_value')),
                          (change.number, propname, encoded_value))

    def migrate_changes(self):
        # if we still have a changes.pck, then we need to migrate it
        changes_pickle = os.path.join(self.basedir, "changes.pck")
        if os.path.exists(changes_pickle):
            if not self.quiet: print "migrating changes.pck to database"

            # 'source' will be an old b.c.changes.ChangeMaster instance, with a
            # .changes attribute
            source = cPickle.load(open(changes_pickle,"rb"))
            styles.doUpgrade()

            if not self.quiet: print " (%d Change objects)" % len(source.changes)

            # first, scan for changes without a number.  If we find any, then we'll
            # renumber the changes sequentially
            have_unnumbered = False
            for c in source.changes:
                if c.revision and c.number is None:
                    have_unnumbered = True
                    break
            if have_unnumbered:
                n = 1
                for c in source.changes:
                    if c.revision:
                        c.number = n
                        n = n + 1

            # insert the changes
            cursor = self.conn.cursor()
            for c in source.changes:
                if not c.revision:
                    continue
                self._addChangeToDatabase(c, cursor)

            # update next_changeid
            max_changeid = max([ c.number for c in source.changes if c.revision ] + [ 0 ])
            cursor.execute("""INSERT into changes_nextid VALUES (%d)""" % (max_changeid+1))

            if not self.quiet:
                print "moving changes.pck to changes.pck.old; delete it or keep it as a backup"
            os.rename(changes_pickle, changes_pickle+".old")
        else:
            c = self.conn.cursor()
            c.execute("""INSERT into changes_nextid VALUES (1)""")

    def set_version(self):
        c = self.conn.cursor()
        c.execute("""INSERT INTO version VALUES (1)""")

