# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import settings
import model
import misc
import time
import datetime
import urllib2
import logging

logging.basicConfig(format='%(asctime)s - %(levelname)s - %(message)s', level=logging.INFO)
BASE_URL = u"http://%s.lianjia.com/" % (settings.CITY)

def get_house_percommunity(communityname):
    url = BASE_URL + u"ershoufang/rs" + urllib2.quote(communityname.encode('utf8')) + "/"
    source_code = misc.get_source_code(url)
    soup = BeautifulSoup(source_code, 'lxml')

    if check_block(soup):
        return
    total_pages = misc.get_sh_total_pages(url)
    
    if total_pages == None:
        row = model.Houseinfo.select().count()
        raise RuntimeError("Finish at %s because total_pages is None" % row)

    for page in range(total_pages):
        if page > 0:
            url_page = BASE_URL + u"ershoufang/d%drs%s/" % (page, urllib2.quote(communityname.encode('utf8')))
            source_code = misc.get_source_code(url_page)
            soup = BeautifulSoup(source_code, 'lxml')

        nameList = soup.findAll("div", {"class":"info"})
        i = 0
        log_progress("GetHouseByCommunitylist", communityname, page+1, total_pages)
        data_source = []
        hisprice_data_source = []
        for name in nameList: # per house loop
            i = i + 1
            info_dict = {}
            try:
                housetitle = name.find("div", {"class":"prop-title"})
                info_dict.update({u'title':housetitle.a.get('title')})
                info_dict.update({u'link':housetitle.a.get('href')})
                info_dict.update({u'houseID':housetitle.a.get('key')})

                houseaddr = name.find("span", {"class":"info-col row1-text"})
                info = houseaddr.get_text().split('|')
                info_dict.update({u'housetype':info[0].strip()})
                info_dict.update({u'square':info[1].strip()})
                info_dict.update({u'floor':info[2].strip()})
                try:
                    info_dict.update({u'direction':info[3].strip()})
                except:
                    info_dict.update({u'direction':''})
                info_dict.update({u'decoration':''})

                housefloor = name.find("span", {"class":"info-col row2-text"})
                detail = housefloor.get_text().split('|')
                info_dict.update({u'years':detail[-1].strip()})

                community = name.find("a", {"class":"laisuzhou"})        
                info_dict.update({u'community':community.span.get_text().strip()})
                info_dict.update({u'followInfo':''})

                tax = name.find("div", {"class":"property-tag-container"})
                info_dict.update({u'taxtype':"".join(tax.get_text().split())})

                totalPrice = name.find("span", {"class":"total-price strong-num"})
                info_dict.update({u'totalPrice':totalPrice.get_text().strip()})

                unitPrice = name.find("span", {"class":"info-col price-item minor"})
                info_dict.update({u'unitPrice':unitPrice.get_text().strip()})
            except:
                continue
            # houseinfo insert into mysql
            data_source.append(info_dict)
            hisprice_data_source.append({"houseID":info_dict["houseID"], "totalPrice":info_dict["totalPrice"]})
            #model.Houseinfo.insert(**info_dict).upsert().execute()
            #model.Hisprice.insert(houseID=info_dict['houseID'], totalPrice=info_dict['totalPrice']).upsert().execute()

        with model.database.atomic():
            model.Houseinfo.insert_many(data_source).upsert().execute()
            model.Hisprice.insert_many(hisprice_data_source).upsert().execute()
        time.sleep(1)

def get_sell_percommunity(communityname):
    url = BASE_URL + u"chengjiao/rs" + urllib2.quote(communityname.encode('utf8')) + "/"
    source_code = misc.get_source_code(url)
    soup = BeautifulSoup(source_code, 'lxml')

    if check_block(soup):
        return
    total_pages = misc.get_sh_total_pages(url)
    
    if total_pages == None:
        row = model.Sellinfo.select().count()
        raise RuntimeError("Finish at %s because total_pages is None" % row)

    for page in range(total_pages):
        if page > 0:
            url_page = BASE_URL + u"chengjiao/d%drs%s/" % (page, urllib2.quote(communityname.encode('utf8')))
            source_code = misc.get_source_code(url_page)
            soup = BeautifulSoup(source_code, 'lxml')
        i = 0
        log_progress("GetSellByCommunitylist", communityname, page+1, total_pages)
        data_source = []
        for name in soup.findAll("div", {"class":"info"}):
            i = i + 1
            info_dict = {}
            try:
                housetitle = name.findAll("div", {"class":"info-row"})[0]
                info_dict.update({u'title':housetitle.a.get('title')})
                info_dict.update({u'link':housetitle.a.get('href')})
                info_dict.update({u'houseID':housetitle.a.get('key')})
                houseinfo = housetitle.get_text().strip().split(' ')
                info_dict.update({u'housetype':houseinfo[1].strip()})
                info_dict.update({u'square':houseinfo[2].strip('').split('\n')[0]})

                houseaddr = name.find("div", {"class":"row1-text"})
                info = houseaddr.get_text().split('|')
                info_dict.update({u'floor':info[0].strip()})
                try:
                    info_dict.update({u'direction':info[1].strip()})
                except:
                    info_dict.update({u'direction':''})
                info_dict.update({u'status':info[2].strip()})

                years = name.find("span", {"class":"c-prop-tag2"})
                info_dict.update({u'years':years.get_text().strip()})

                community = name.find("span", {"class":"cj-text"})      
                info_dict.update({u'community':community.get_text().strip()})

                totalPrice = name.find("span", {"class":"strong-num"})
                info_dict.update({u'totalPrice':totalPrice.get_text().strip()})

                unitPrice = name.find("div", {"class":"info-col price-item minor"})
                info_dict.update({u'unitPrice':unitPrice.get_text().strip()})

                source = name.find("div", {"class":"info-col deal-item minor"})
                info_dict.update({u'source':source.get_text().strip()})

                dealdate = name.find("div", {"class":"info-col deal-item main strong-num"})
                info_dict.update({u'dealdate':dealdate.get_text().strip().replace('.','-')})

            except:
                continue
            # Sellinfo insert into mysql
            data_source.append(info_dict)
            #model.Sellinfo.insert(**info_dict).upsert().execute()

        with model.database.atomic():
            model.Sellinfo.insert_many(data_source).upsert().execute()
        time.sleep(1)

def get_community_perregion(regionname=u'pudong'):
    url = BASE_URL + u"xiaoqu/" + regionname +"/"
    source_code = misc.get_source_code(url)
    soup = BeautifulSoup(source_code, 'lxml')

    if check_block(soup):
        return
    total_pages = misc.get_sh_total_pages(url)
    
    if total_pages == None:
        row = model.Community.select().count()
        raise RuntimeError("Finish at %s because total_pages is None" % row)

    for page in range(total_pages):
        if page > 0:
            url_page = BASE_URL + u"xiaoqu/" + regionname +"/d%d/" % page
            source_code = misc.get_source_code(url_page)
            soup = BeautifulSoup(source_code, 'lxml')

        nameList = soup.findAll("div", {"class":"info-panel"})
        i = 0
        log_progress("GetCommunityByRegionlist", regionname, page+1, total_pages)
        data_source = []
        for name in nameList: # Per house loop
            i = i + 1
            info_dict = {}
            try:
                communitytitle = name.find("a", {"name":"selectDetail"})
                title = communitytitle.get_text().strip('\n')
                link = communitytitle.get('href')
                id = communitytitle.get('key')
                info_dict.update({u'title':title})
                info_dict.update({u'link':link})
                info_dict.update({u'id':id})

                district = name.find("a", {"class":"ad"})
                info_dict.update({u'district':district.get_text()})
                
                cons = name.find("div", {"class":"con"})
                bizcircle = cons.findAll("a")
                info_dict.update({u'bizcircle':bizcircle[1].get_text().strip()})

                try:
                    tagList = name.find("span", {"class":"fang-subway-ex"})
                    info_dict.update({u'tagList':tagList.get_text().strip()})
                except:
                    info_dict.update({u'tagList':''})

                onsale = name.find("span", {"class":"num"})
                info_dict.update({u'onsale':onsale.get_text().strip()})

                price = name.find("div", {"class":"price"})
                info_dict.update({u'price':price.span.get_text().strip()})

                communityinfo = get_communityinfo_by_url(link)
                for key, value in communityinfo.iteritems():
                    info_dict.update({key:value})

            except:
                continue
            # communityinfo insert into mysql
            data_source.append(info_dict)
            #model.Community.insert(**info_dict).upsert().execute()

        with model.database.atomic():
            model.Community.insert_many(data_source).upsert().execute()
        time.sleep(1)

def get_rent_percommunity(communityname):
    url = BASE_URL + u"zufang/rs" + urllib2.quote(communityname.encode('utf8')) + "/"
    source_code = misc.get_source_code(url)
    soup = BeautifulSoup(source_code, 'lxml')

    if check_block(soup):
        return
    total_pages = misc.get_sh_total_pages(url)

    if total_pages == None:
        row = model.Rentinfo.select().count()
        raise RuntimeError("Finish at %s because total_pages is None" % row)

    for page in range(total_pages):
        if page > 0:
            url_page = BASE_URL + u"rent/d%drs%s/" % (page, urllib2.quote(communityname.encode('utf8')))
            source_code = misc.get_source_code(url_page)
            soup = BeautifulSoup(source_code, 'lxml')
        i = 0
        log_progress("GetRentByCommunitylist", communityname, page+1, total_pages)
        data_source = []
        nameList = soup.findAll("div", {"class":"info-panel"})
        for name in nameList:
            i = i + 1
            info_dict = {}
            try:
                info = name.find("a", {"name" : "selectDetail"})
                info_dict.update({u'title':info.get('title')})
                info_dict.update({u'link':info.get('href')})
                info_dict.update({u'houseID':info.get('key')})


                where = name.find("div", {"class":"where"})
                wheres = where.find_all("span")
                info_dict.update({u'region':wheres[0].get_text().strip()})
                info_dict.update({u'zone':wheres[1].get_text().strip()})
                info_dict.update({u'meters':wheres[2].get_text().strip()})

                other = name.find("div", {"class":"con"})
                info_dict.update({u'other':"".join(other.get_text().split())})

                info_dict.update({u'subway':""})
                info_dict.update({u'decoration':""})
                info_dict.update({u'heating':""})

                price = name.find("div", {"class":"price"})
                info_dict.update({u'price':int(price.span.get_text().strip())})

                pricepre = name.find("div", {"class":"price-pre"})
                info_dict.update({u'pricepre':"".join(pricepre.get_text().split())})

            except:
                continue
            # Rentinfo insert into mysql
            data_source.append(info_dict)
            #model.Rentinfo.insert(**info_dict).upsert().execute()

            with model.database.atomic():
                model.Rentinfo.insert_many(data_source).upsert().execute()
            time.sleep(1)

def get_house_perregion(district):
    url = BASE_URL + u"ershoufang/%s/" % district
    source_code = misc.get_source_code(url)
    soup = BeautifulSoup(source_code, 'lxml')
    if check_block(soup):
        return
    total_pages = misc.get_sh_total_pages(url)
    if total_pages == None:
        row = model.Houseinfo.select().count()
        raise RuntimeError("Finish at %s because total_pages is None" % row)

    for page in range(total_pages):
        if page > 0:
            url_page = BASE_URL + u"ershoufang/%s/d%d/" % (district, page)
            source_code = misc.get_source_code(url_page)
            soup = BeautifulSoup(source_code, 'lxml')
        i = 0
        log_progress("GetHouseByRegionlist", district, page+1, total_pages)
        data_source = []
        hisprice_data_source = []
        nameList = soup.findAll("div", {"class":"info"})
        for name in nameList: # per house loop
            i = i + 1
            info_dict = {}
            try:
                housetitle = name.find("div", {"class":"prop-title"})
                info_dict.update({u'title':housetitle.a.get('title')})
                info_dict.update({u'link':housetitle.a.get('href')})
                info_dict.update({u'houseID':housetitle.a.get('key')})

                houseaddr = name.find("span", {"class":"info-col row1-text"})
                info = houseaddr.get_text().split('|')
                info_dict.update({u'housetype':info[0].strip()})
                info_dict.update({u'square':info[1].strip()})
                info_dict.update({u'floor':info[2].strip()})
                try:
                    info_dict.update({u'direction':info[3].strip()})
                except:
                    info_dict.update({u'direction':''})
                info_dict.update({u'decoration':''})

                housefloor = name.find("span", {"class":"info-col row2-text"})
                detail = housefloor.get_text().split('|')
                info_dict.update({u'years':detail[-1].strip()})

                community = name.find("a", {"class":"laisuzhou"})        
                info_dict.update({u'community':community.span.get_text().strip()})
                info_dict.update({u'followInfo':''})

                tax = name.find("div", {"class":"property-tag-container"})
                info_dict.update({u'taxtype':"".join(tax.get_text().split())})

                totalPrice = name.find("span", {"class":"total-price strong-num"})
                info_dict.update({u'totalPrice':totalPrice.get_text().strip()})

                unitPrice = name.find("span", {"class":"info-col price-item minor"})
                info_dict.update({u'unitPrice':unitPrice.get_text().strip()})
            except:
                continue
            # houseinfo insert into mysql
            data_source.append(info_dict)
            hisprice_data_source.append({"houseID":info_dict["houseID"], "totalPrice":info_dict["totalPrice"]})
            #model.Houseinfo.insert(**info_dict).upsert().execute()
            #model.Hisprice.insert(houseID=info_dict['houseID'], totalPrice=info_dict['totalPrice']).upsert().execute()

            with model.database.atomic():
                model.Houseinfo.insert_many(data_source).upsert().execute()
                model.Hisprice.insert_many(hisprice_data_source).upsert().execute()
            time.sleep(1)

def get_rent_perregion(district):
    url = BASE_URL + u"zufang/%s/" % district
    source_code = misc.get_source_code(url)
    soup = BeautifulSoup(source_code, 'lxml')
    if check_block(soup):
        return
    total_pages = misc.get_sh_total_pages(url)
    if total_pages == None:
        row = model.Rentinfo.select().count()
        raise RuntimeError("Finish at %s because total_pages is None" % row)

    for page in range(total_pages):
        if page > 0:
            url_page = BASE_URL + u"zufang/%s/d%d/" % (district, page)
            source_code = misc.get_source_code(url_page)
            soup = BeautifulSoup(source_code, 'lxml')
        i = 0
        log_progress("GetRentByRegionlist", district, page+1, total_pages)
        data_source = []
        nameList = soup.findAll("div", {"class":"info-panel"})
        for name in nameList:
            i = i + 1
            info_dict = {}
            try:
                info = name.find("a", {"name" : "selectDetail"})
                info_dict.update({u'title':info.get('title')})
                info_dict.update({u'link':info.get('href')})
                info_dict.update({u'houseID':info.get('key')})


                where = name.find("div", {"class":"where"})
                wheres = where.find_all("span")
                info_dict.update({u'region':wheres[0].get_text().strip()})
                info_dict.update({u'zone':wheres[1].get_text().strip()})
                info_dict.update({u'meters':wheres[2].get_text().strip()})

                other = name.find("div", {"class":"con"})
                info_dict.update({u'other':"".join(other.get_text().split())})

                info_dict.update({u'subway':""})
                info_dict.update({u'decoration':""})
                info_dict.update({u'heating':""})

                price = name.find("div", {"class":"price"})
                info_dict.update({u'price':int(price.span.get_text().strip())})

                pricepre = name.find("div", {"class":"price-pre"})
                info_dict.update({u'pricepre':"".join(pricepre.get_text().split())})

            except:
                continue
            # Rentinfo insert into mysql
            data_source.append(info_dict)
            #model.Rentinfo.insert(**info_dict).upsert().execute()

        with model.database.atomic():
            model.Rentinfo.insert_many(data_source).upsert().execute()
        time.sleep(1)

def get_communityinfo_by_url(url):
    source_code = misc.get_source_code(BASE_URL+url)
    soup = BeautifulSoup(source_code, 'lxml')

    if check_block(soup):
        return

    communityinfos = soup.findAll("div", {"class":"col-2 clearfix"})
    res = {}
    for info in communityinfos:
        try:
            infos = info.findAll("li")
            housetype = infos[0].find("span", {"class":"other"})
            year = infos[1].find("span", {"class":"other"})
            cost = infos[2].find("span", {"class":"other"})
            service = infos[3].span.find(text=True, recursive=False)
            company = infos[4].span.find(text=True, recursive=False)
            res.update({'housetype': housetype.get_text().strip()})
            res.update({'year': year.get_text().strip()})
            res.update({'cost': cost.get_text().strip()})
            res.update({'service': service.strip()})
            res.update({'company': company.strip()})

        except:
            continue
    return res

def check_block(soup):
    if soup.title.string == "414 Request-URI Too Large":
        logging.error("Lianjia block your ip, please verify captcha manually at lianjia.com")
        return True
    return False

def log_progress(function, address, page, total):
    logging.info("Progress: %s %s: current page %d total pages %d" %(function, address, page, total))