#!/usr/bin/env python

import sys, time, logging, signal, threading
import os, shutil, Queue, fnmatch, multiprocessing
from signal import SIGTERM, SIGKILL
from datetime import datetime, date
from daemon import Daemon

class FileReader(multiprocessing.Process):

  def __init__(self, name, loghandler, work_queue, sql_queue):
    multiprocessing.Process.__init__(self)
    self.log = loghandler
    self.work_queue = work_queue
    self.sql_queue = sql_queue
    self.kill_message = False
    self.name = name

  def shutdown(self, *args, **kwarg):
    self.kill_message = True
    self.log.info("shutting down {0}".format(self.name))  

  def run(self):
    pidmsg = "file worker pid: {0}".format(os.getpid())
    self.log.info(pidmsg)
    signal.signal(SIGTERM, self.shutdown)
    run_count = 0
    while not self.kill_message:
      try:
        run_count += 1
        task = self.work_queue.get()
        task_message = "data from {0}".format(task)
        self.sql_queue.put(task_message)
      except KeyboardInterrupt:
        break
      if self.work_queue.empty():
        break

    stats_msg = "worker: {0} processed {1}".format(self.name, run_count)
    self.log.info(stats_msg)
    self.log.info("process {0} killed".format(self.name))

class procDisk(multiprocessing.Process):

  def __init__(self, name, loghandler, location, work_queue):
    multiprocessing.Process.__init__(self)
    self.kill_message = False
    self.log = loghandler
    self.location = location
    self.sql_queue = work_queue
    self.last_check = datetime(1970,01,01)
    self.file_queue = multiprocessing.Queue()
    self.name = name
    self.worker_count = 0

  def check_disk(self):
    found_files = False
    for root, dirs, files in os.walk(self.location):
      for filename in fnmatch.filter(files,"*.tar.gz"): 
        # check to see if the file is newer than the last "run" time
        # if it is, its new -- otherwise it should alread by in the queue
        mod_time = datetime.fromtimestamp(os.path.getmtime(os.path.join(root, filename)))
        if mod_time >= self.last_check:
          self.file_queue.put(os.path.join(root, filename))
          found_files = True

    if found_files:
      self.last_check = datetime.now()

  def shutdown(self, *args, **kwarg):
    self.kill_message = True
    self.log.info("shutting down {0}".format(self.name))

  def run(self):
    pidmsg = "disk proc pid: {0}".format(os.getpid())
    self.log.info(pidmsg)
    worker_spawns = []
    signal.signal(SIGTERM, self.shutdown)

    while not self.kill_message:
      try:
        self.check_disk()
     
        if not self.file_queue.empty():
          if self.worker_count <= 4:
            self.log.info("spawning process...")
            worker = FileReader("FileSpawn " + str(self.worker_count), self.log, self.file_queue, self.sql_queue)
            worker.start()
            worker_spawns.append(worker)
            self.worker_count += 1

        # output some stats
        stats_msg = "file queue: {0} sql queue: {1} workers: {2}".format(self.file_queue.qsize(), self.sql_queue.qsize(), self.worker_count)
        self.log.info(stats_msg) 
        time.sleep(20)
      except KeyboardInterrupt:
        break

    for child in worker_spawns:
      self.log.info("Killing worker {0}".format(child.name))
      child.terminate()

    self.log.info("{0} killed".format(self.name))  
    
class procSQL(multiprocessing.Process):
  def __init__(self, name, loghandler, work_queue):
    multiprocessing.Process.__init__(self)
    self.log = loghandler
    self.name = name
    self.kill_message = False
    self.sql_queue = work_queue

  def shutdown(self, *args, **kwargs):
    self.kill_message = True
    self.log.info("shutting down")

  def run(self):
    pidmsg = "procSQL PID: {0}".format(os.getpid())
    self.log.info(pidmsg)
    signal.signal(SIGTERM, self.shutdown)
    while not self.kill_message:
      try:
        now = datetime.now()
        task = self.sql_queue.get()
        stats_msg = "SQL: {0}".format(self.sql_queue.qsize())
        self.log.info(stats_msg)
        time.sleep(10)
      except KeyboardInterrupt:
        break
    self.log.info(
      ("{0} killed.".format(self.name))
    )

class MainDaemon(Daemon):
  '''
  implement the daemon
  '''
  def __init__(self, *args, **kwargs):
    super(MainDaemon,self).__init__(*args, **kwargs)
    self.stopping = False
    self.main_proc = []
    self.qSQLdata = multiprocessing.Queue()

  def set_options(self, options, arguments):
    self.options = options
    self.arguments = arguments

  def set_stop(self, *args, **kwargs):
    self.stopping = True
    self.log.info('set stop! going to stop now')

  def is_stopping(self):
    return self.stopping

  def run(self):
    pidmsg = "MainDaemon PID: {0}".format(os.getpid())
    self.log.info(pidmsg)
    signal.signal(SIGTERM, self.set_stop)
  
    # start our two main threads
    qdisk = procDisk("Disk Check Process", self.log, self.arguments[0], self.qSQLdata)
    qsqldata = procSQL("SQL Data Process", self.log, self.qSQLdata)
    self.main_proc.append(qdisk)
    self.main_proc.append(qsqldata)

    for child in self.main_proc:
      child.start()

    # main loop
    while not self.is_stopping():
      try:
        time.sleep(60)
      except KeyboardInterrupt:
        self.log.info("CTRL-C received - quitting")
        self.set_stop()
        break

    # SIGTERM received; quit
    for child in self.main_proc:
      self.log.info("killing {0}".format(child.name))
      child.terminate()
    self.log.info('Stopping MainDaemon')