Ok, I think the plugin was installed when my predecessor tried to perform
the migration. I can try to go through it again.
Any idea why Bitten is slowing things down a lot? I know we have a patch
that was applied to add some desired functionality to Bitten, like
keep-alive communication, network timeout retries, etc.
I've attached our patch for reference.
Thanks!!
Tomas
On Wednesday, March 11, 2015 at 10:35:02 AM UTC-7, RjOllos wrote:
>
> On Wed, Mar 11, 2015 at 10:30 AM, <tol...@gmail.com <javascript:>> wrote:
>
>> I'm strongly considering switching to PostgreSQL. Is it a straightforward
>> transition?
>> Can you point me to a link or describe how you performed the switch?
>>
>> A lot of my users are not here anymore. Would it help to delete their
>> user accounts?
>> Would that affect any tickets they're associated with?
>>
>> Thanks!
>>
>
> The TracMigratePlugin, which you show is already installed in your plugin
> list, will help with the migration. I suggest going with PostgreSQL since
> there are fewer caveats involved with how your configure the database. If
> you decide to use MySQL, I suggest upgrading to Trac 1.0.4 first.
>
> http://trac-hacks.org/wiki/TracMigratePlugin
>
> I don't think it will help to delete users in this case. The issues with
> SQLite come down to how much load there is on your system.
>
>
--
You received this message because you are subscribed to the Google Groups "Trac
Users" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to trac-users+unsubscr...@googlegroups.com.
To post to this group, send email to trac-users@googlegroups.com.
Visit this group at http://groups.google.com/group/trac-users.
For more options, visit https://groups.google.com/d/optout.
Index: bitten/admin.py
===================================================================
--- bitten/admin.py (revision 1021)
+++ bitten/admin.py (working copy)
@@ -55,6 +55,12 @@
build_all and 'yes' or 'no')
changed = True
+ build_order = 'build_order' in req.args
+ if build_order != master.build_order:
+ self.config['bitten'].set('build_order',
+ build_order and 'yes' or 'no')
+ changed = True
+
adjust_timestamps = 'adjust_timestamps' in req.args
if adjust_timestamps != master.adjust_timestamps:
self.config['bitten'].set('adjust_timestamps',
Index: bitten/master.py
===================================================================
--- bitten/master.py (revision 1021)
+++ bitten/master.py (working copy)
@@ -56,6 +56,10 @@
"""Whether to request builds of older revisions even if a younger
revision has already been built.""")
+ build_order = BoolOption('bitten', 'build_order', False, doc=
+ """Whether to build older revisions first even if a younger revision
+ is available.""")
+
stabilize_wait = IntOption('bitten', 'stabilize_wait', 0, doc=
"""The time in seconds to wait for the repository to stabilize before
queuing up a new build. This allows time for developers to check in
@@ -161,6 +165,7 @@
def _process_build_creation(self, req, slave_token):
queue = BuildQueue(self.env, build_all=self.build_all,
+ build_order=self.build_order,
stabilize_wait=self.stabilize_wait,
timeout=self.slave_timeout)
try:
Index: bitten/model.py
===================================================================
--- bitten/model.py (revision 1021)
+++ bitten/model.py (working copy)
@@ -546,7 +546,8 @@
fetch = classmethod(fetch)
def select(cls, env, config=None, rev=None, platform=None, slave=None,
- status=None, db=None, min_rev_time=None, max_rev_time=None):
+ status=None, db=None, min_rev_time=None, max_rev_time=None,
+ order=False):
"""Retrieve existing builds from the database that match the specified
criteria.
"""
@@ -573,10 +574,17 @@
else:
where = ""
+ # The keyword DESC reverts to descending order
cursor = db.cursor()
- cursor.execute("SELECT id FROM bitten_build %s "
+ if order:
+ cursor.execute("SELECT id FROM bitten_build %s "
+ "ORDER BY rev_time,config,slave"
+ % where, [wc[1] for wc in where_clauses])
+ else:
+ cursor.execute("SELECT id FROM bitten_build %s "
"ORDER BY rev_time DESC,config,slave"
% where, [wc[1] for wc in where_clauses])
+
for (id,) in cursor:
yield Build.fetch(env, id)
select = classmethod(select)
Index: bitten/queue.py
===================================================================
--- bitten/queue.py (revision 1021)
+++ bitten/queue.py (working copy)
@@ -97,11 +97,12 @@
repository revisions that need to be built.
"""
- def __init__(self, env, build_all=False, stabilize_wait=0, timeout=0):
+ def __init__(self, env, build_all=False, build_order=False, stabilize_wait=0, timeout=0):
"""Create the build queue.
:param env: the Trac environment
:param build_all: whether older revisions should be built
+ :param build_order: whether to build older revisions first
:param stabilize_wait: The time in seconds to wait before considering
the repository stable to create a build in the queue.
:param timeout: the time in seconds after which an in-progress build
@@ -111,6 +112,7 @@
self.env = env
self.log = env.log
self.build_all = build_all
+ self.build_order = build_order
self.stabilize_wait = stabilize_wait
self.timeout = timeout
@@ -128,6 +130,11 @@
:rtype: `Build`
"""
self.log.debug('Checking for pending builds...')
+ order = self.build_order
+ if order:
+ self.log.debug('Retrieving oldest revisions first')
+ else:
+ self.log.debug('Retrieving newest revisions first')
db = self.env.get_db_cnx()
repos = self.env.get_repository()
@@ -141,7 +148,8 @@
platforms = [p.id for p in self.match_slave(name, properties)]
builds_to_delete = []
build_found = False
- for build in Build.select(self.env, status=Build.PENDING, db=db):
+ for build in Build.select(self.env, status=Build.PENDING, db=db,
+ order=order):
if self.should_delete_build(build, repos):
self.log.info('Scheduling build %d for deletion', build.id)
builds_to_delete.append(build)
@@ -224,7 +232,6 @@
for config in BuildConfig.select(self.env, db=db):
platforms = []
for platform, rev, build in collect_changes(repos, config, db):
-
if not self.build_all and platform.id in platforms:
# We've seen this platform already, so these are older
# builds that should only be built if built_all=True
Index: bitten/slave.py
===================================================================
--- bitten/slave.py (revision 1021)
+++ bitten/slave.py (working copy)
@@ -130,10 +130,12 @@
class KeepAliveThread(threading.Thread):
"A thread to periodically send keep-alive messages to the master"
- def __init__(self, opener, build_url, single_build, keepalive_interval):
+ def __init__(self, opener, build_url, single_build, keepalive_interval,
+ retry_count):
threading.Thread.__init__(self, None, None, "KeepaliveThread")
self.build_url = build_url
self.keepalive_interval = keepalive_interval
+ self.retry_count = retry_count
self.single_build = single_build
self.last_keepalive = int(time.time())
self.kill = False
@@ -151,18 +153,31 @@
}
log.debug('Sending %s request to %r', method, url)
req = SaneHTTPRequest(method, url, body, headers or {})
- try:
- return self.opener.open(req)
- except urllib2.HTTPError, e:
- # a conflict error lets us know that we've been
- # invalidated. Ideally, we'd engineer something to stop any
- # running steps in progress, but killing threads is tricky
- # stuff. For now, we'll wait for whatever's going
- # on to stop, and the main thread'll figure out that we've
- # been invalidated.
- log.warning('Server returned keepalive error %d: %s', e.code, e.msg)
- except:
- log.warning('Server returned unknown keepalive error')
+ #try_pause = 10
+ for try_number in range(self.retry_count+1):
+ try:
+ return self.opener.open(req)
+ except urllib2.HTTPError as e:
+ # a conflict error lets us know that we've been
+ # invalidated. Ideally, we'd engineer something to stop any
+ # running steps in progress, but killing threads is tricky
+ # stuff. For now, we'll wait for whatever's going
+ # on to stop, and the main thread'll figure out that we've
+ # been invalidated.
+ if e.code == 500:
+ log.warning('Server timed out during keepalive')
+ continue
+ elif try_number < self.retry_count:
+ try_pause = 2**(try_number+3)
+ log.warning('Retry [%i more, waiting %i seconds]' % (self.retry_count-try_number, try_pause))
+ time.sleep(try_pause)
+ continue
+ else:
+ log.warning('Server returned keepalive error %d: %s', e.code, e.msg)
+ raise
+ return e
+ except:
+ log.warning('Server returned unknown keepalive error')
def run(self):
log.debug('Keepalive thread starting.')
@@ -190,7 +205,8 @@
keep_files=False, single_build=False,
poll_interval=300, keepalive_interval = 60,
username=None, password=None,
- dump_reports=False, no_loop=False, form_auth=False):
+ dump_reports=False, no_loop=False, form_auth=False,
+ retry_count=0):
"""Create the build slave instance.
:param urls: a list of URLs of the build masters to connect to, or a
@@ -221,6 +237,7 @@
of whether a build is done or not
:param form_auth: login using AccountManager HTML form instead of
HTTP authentication for all urls
+ :param retry_count: number of times to retry network requests
"""
self.local = len(urls) == 1 and not urls[0].startswith('http://') \
and not urls[0].startswith('https://')
@@ -251,6 +268,7 @@
self.cookiejar = cookielib.CookieJar()
self.username = username \
or self.config['authentication.username'] or ''
+ self.retry_count = retry_count
if not self.local:
self.password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
@@ -277,23 +295,29 @@
def request(self, method, url, body=None, headers=None):
log.debug('Sending %s request to %r', method, url)
req = SaneHTTPRequest(method, url, body, headers or {})
- try:
- resp = self.opener.open(req)
- if not hasattr(resp, 'code'):
- resp.code = 200
- return resp
- except urllib2.HTTPError, e:
- if e.code >= 300:
- if hasattr(e, 'headers') and \
- e.headers.getheader('Content-Type', ''
- ).startswith('text/plain'):
- content = e.read()
- else:
- content = 'no message available'
- log.debug('Server returned error %d: %s (%s)',
- e.code, e.msg, content)
- raise
- return e
+ #try_pause = 10
+ for try_number in range(self.retry_count+1):
+ try:
+ resp = self.opener.open(req)
+ if not hasattr(resp, 'code'):
+ resp.code = 200
+ return resp
+ except urllib2.HTTPError as e:
+ if e.code >= 300:
+ if hasattr(e, 'headers') and \
+ e.headers.getheader('Content-Type', '').startswith('text/plain'):
+ content = e.read()
+ else:
+ content = 'no message available'
+ if try_number < self.retry_count:
+ try_pause = 2**(try_number+3)
+ log.warning('Retry [%i more, waiting %i seconds]' % (self.retry_count-try_number, try_pause))
+ time.sleep(try_pause)
+ continue
+ else:
+ log.warning('Server returned error %d: %s', e.code, e.msg)
+ raise
+ return e
def run(self):
if self.local:
@@ -418,7 +442,8 @@
try:
if not self.local:
keepalive_thread = KeepAliveThread(self.opener, build_url,
- self.single_build, self.keepalive_interval)
+ self.single_build, self.keepalive_interval,
+ self.retry_count)
keepalive_thread.start()
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
self.config)
@@ -561,11 +586,13 @@
from getpass import getpass
parser.values.password = getpass('Password: ')
parser.add_option('-P', '--ask-password', action='callback',
- callback=_ask_password, help='Prompt for password')
- parser.add_option('--form-auth', action='store_true',
+ callback=_ask_password, help='prompt for password')
+ parser.add_option('-a', '--form-auth', action='store_true',
dest='form_auth',
help='login using AccountManager HTML form instead of '
'HTTP authentication for all urls')
+ parser.add_option('-r', '--retry-count', dest='retry_count', type='int',
+ help='number of times to retry network requests')
group = parser.add_option_group('building')
group.add_option('-d', '--work-dir', action='store', dest='work_dir',
@@ -588,7 +615,10 @@
help='don\'t report results back to master')
group.add_option('-i', '--interval', dest='interval', metavar='SECONDS',
type='int', help='time to wait between requesting builds')
- group.add_option('-b', '--keepalive_interval', dest='keepalive_interval', metavar='SECONDS', type='int', help='time to wait between keepalive heartbeats')
+ group.add_option('-b', '--keepalive-interval', dest='keepalive_interval',
+ metavar='SECONDS', type='int', help='time to wait between '
+ 'keepalive heartbeats')
+
group = parser.add_option_group('logging')
group.add_option('-l', '--log', dest='logfile', metavar='FILENAME',
help='write log messages to FILENAME')
@@ -602,7 +632,7 @@
parser.set_defaults(dry_run=False, keep_files=False,
loglevel=logging.INFO, single_build=False, no_loop=False,
dump_reports=False, interval=300, keepalive_interval=60,
- form_auth=False)
+ form_auth=False, retry_count=0)
options, args = parser.parse_args()
if len(args) < 1:
@@ -624,6 +654,9 @@
handler.setFormatter(formatter)
logger.addHandler(handler)
+ log.debug("Option retry_count is set to %s." % options.retry_count)
+ log.debug("Option interval is set to %s." % options.interval)
+ log.debug("Option keepalive_interval is set to %s." % options.keepalive_interval)
log.info("Slave launched at %s" % \
datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
@@ -639,7 +672,8 @@
keepalive_interval=options.keepalive_interval,
username=options.username, password=options.password,
dump_reports=options.dump_reports,
- form_auth=options.form_auth)
+ form_auth=options.form_auth,
+ retry_count=options.retry_count)
try:
exit_code = slave.run()
except KeyboardInterrupt:
Index: bitten/templates/bitten_admin_master.html
===================================================================
--- bitten/templates/bitten_admin_master.html (revision 1021)
+++ bitten/templates/bitten_admin_master.html (working copy)
@@ -27,6 +27,17 @@
</p>
<div class="field">
<label>
+ <input type="checkbox" id="build_order" name="build_order"
+ checked="${master.build_order and 'checked' or None}" />
+ Build older first
+ </label>
+ </div>
+ <p class="hint">
+ Whether to build older revisions first even when a more recent
+ revision is available.
+ </p>
+ <div class="field">
+ <label>
<input type="checkbox" id="adjust_timestamps"
name="adjust_timestamps"
checked="${master.adjust_timestamps and 'checked'
Index: bitten/tests/admin.py
===================================================================
--- bitten/tests/admin.py (revision 1021)
+++ bitten/tests/admin.py (working copy)
@@ -90,6 +90,7 @@
self.assertEqual(0, master.stabilize_wait)
assert not master.adjust_timestamps
assert not master.build_all
+ assert not master.build_order
self.assertEqual('log/bitten', master.logs_dir)
def test_process_config_changes(self):
@@ -113,6 +114,7 @@
self.assertEqual(60, section.getint('slave_timeout'))
self.assertEqual(True, section.getbool('adjust_timestamps'))
self.assertEqual(False, section.getbool('build_all'))
+ self.assertEqual(False, section.getbool('build_order'))
class BuildConfigurationsAdminPageProviderTestCase(unittest.TestCase):
Index: setup.cfg
===================================================================
--- setup.cfg (revision 1021)
+++ setup.cfg (working copy)
@@ -1,6 +1,5 @@
[egg_info]
tag_build = dev
-tag_svn_revision = true
[unittest]
xml_output = build/test-results.xml
