# -*- coding: utf-8 -*-

import re
import urllib
import urllib.request
from string import Template

from apscheduler.jobstores.sqlalchemy import SQLAlchemyJobStore
from scrapy import cmdline
from datetime import datetime
import time
import os
from scrapy.cmdline import execute
import sys
from apscheduler.triggers.cron import CronTrigger
from apscheduler.schedulers.background import BackgroundScheduler


def download(url):
    """Simple downloader"""
    return urllib.request.urlopen(url).read()


def crawl_sitemap():
    # download the sitemap file
    str1 = Template('https://www.fmo.org.hk/price?id=8&path=12_43_56&page=${pageNum}&per-page=10')
    siteUrl = str1.substitute(pageNum='1')
    print(siteUrl)
    sitemap = download(siteUrl)
    rests = str(sitemap, encoding="utf-8")
    print(rests)


def job_function():
    print('running crawler task, getting data from HongKong...... ')
    # ValueError: signal only works in main thread
    # cmdline.execute("scrapy crawl hongkong --nolog".split())
    # execute(["scrapy", "crawl", "hongkong"])
    os.system('scrapy crawl hongkong --nolog')


if __name__ == '__main__':
    # crawl_sitemap()
    scheduler = BackgroundScheduler(
        jobstores={'sqlite': SQLAlchemyJobStore(url='sqlite:////root/crawler.db')},
        job_defaults={'misfire_grace_time': 15 * 60},
    )
    scheduler.add_job(job_function, CronTrigger.from_crontab('10 22 * * *'))
    scheduler.start()
    print('Crawler of HongKong has been started by apscheduler.')
    try:
        while True:
            # print('========scheduler has been started...')
            time.sleep(5)
    except (KeyboardInterrupt, SystemExit):
        scheduler.shutdown()