#!/usr/bin/env python3
import re
import urllib.request
from random import choice
from os.path import isfile
from time import gmtime

if __name__ == '__main__':
    userAgents = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36', 'Opera/9.63 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1', 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11', 	'Mozilla/5.0 (Windows NT x.y; Win64; x64; rv:10.0) Gecko/20100101 Firefox/10.0', 'Mozilla/5.0 (Macintosh; PPC Mac OS X x.y; rv:10.0) Gecko/20100101 Firefox/10.0', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0')
    categories = ('Lebensmittel_67299','Mutter--Kind_67325','Heim--Garten_67334','Gesundheit--Kosmetik_67391','Elektronik--Technik_67357','Bekleidung--Schmuck_67426','Sport--Freizeit_67399','Fahrzeuge_67414','Dienstleistungen_67502')
    offers = []
    
    last = len(offers)
    offerPattern = re.compile('<div class=\"design-ad-name\">\n\s+<a href=\"\/de\/anzeige\/(.*)\" title')
    offerIdPattern = re.compile('_(\d*)$')
    quantityPattern = re.compile('marketplaceArticleQuanitityValue\">(\d*)</')
    
    for cat in categories:
        for i in range(1,65535):
            url = "https://markt.kadari.net/de/produkte/" + cat +  "__RUNTIME_DESC_" + str(i) + "_LIST"
            req = urllib.request.Request(
                url, 
                data=None, 
                headers={
                'User-Agent': choice(userAgents)
                }
            )
            f = urllib.request.urlopen(req)
            content = f.read().decode('utf-8')
            offers.extend(offerPattern.findall(content))
            if (len(offers) == last):
                break
            else:
                last = len(offers)
    
    offers = list(set(offers))
    stock = {}
    
    for offer in offers:
        url = "https://markt.kadari.net/de/anzeige/" + offer
        req = urllib.request.Request(
            url, 
            data=None, 
            headers={
            'User-Agent': choice(userAgents)
            }
        )
        f = urllib.request.urlopen(req)
        content = f.read().decode('utf-8','replace')
        offerId = int(offerIdPattern.search(offer).groups()[0])
        quantity = int(quantityPattern.search(content).groups()[0])
        if (quantity > 0):
            stock[offerId] = quantity
        else:
            stock[offerId] = 0
    
    lines = []
    headline = []
    oldfilename = ""
    t = gmtime()
    timestring = "{:02d}".format(t.tm_mday) + "." + "{:02d}".format(t.tm_mon) + "." + "{:04d}".format(t.tm_year) +  " " + "{:02d}".format(t.tm_hour) + ":" + "{:02d}".format(t.tm_min)
      
    if isfile("history.csv"):
        with open("history.csv",'r', encoding='utf-8') as historyFile:
            lineNumber = 0
            for historyLine in historyFile:
                if lineNumber == 0 and historyLine != "sep=;\n":
                    lineNumber = 1
                if lineNumber == 1:
                    headline.extend(historyLine.split(';'))
                elif lineNumber > 1:
                    lines.append(historyLine)
                lineNumber += 1
        while headline[0] == '':
            del headline[0]
        while headline[-1] == '\n':
            del headline[-1]
        headline = [int(i) for i in headline]
        toadd = list(set(stock.keys()).difference(set(headline)))        
        oldstamp = lines[-1][:16]
        oldfilename = oldstamp[6:10] + "-" + oldstamp[3:5] + "-" + oldstamp[0:2] + "_" + oldstamp[11:13] + "-" + oldstamp[14:16] + "_history.csv"
        print(oldfilename)
        if not isfile(oldfilename):
            with open(oldfilename,'w', encoding='utf-8') as oldHistoryFile:
                if lines[0] != "sep=;\n":
                    oldHistoryFile.write("sep=;\n")
                    oldHistoryFile.write(";")
                for id in headline:
                    oldHistoryFile.write(str(id))
                    oldHistoryFile.write(";")
                for line in lines:
                    oldHistoryFile.write(line)
        with open("history.csv",'w', encoding='utf-8') as historyFile:
            historyFile.write("sep=;\n")
            historyFile.write(";")
            for id in headline:
                historyFile.write(str(id))
                historyFile.write(";")
            for id in toadd:
                historyFile.write(str(id))
                historyFile.write(";")
            historyFile.write("\n")
            for line in lines:
                historyFile.write(line)
            historyFile.write(timestring)
            historyFile.write(";")
            for id in headline:
                if id in stock.keys():
                    historyFile.write(str(stock[int(id)]))
                    historyFile.write(";")
                else:
                    historyFile.write(";")
            for id in toadd:
                historyFile.write(str(stock[int(id)]))
                historyFile.write(";")            
            historyFile.write("\n")
    else:
        with open("history.csv",'w') as historyFile:
            historyFile.write("sep=;\n")
            historyFile.write(";")
            for id in stock.keys():
                historyFile.write(str(id))
                historyFile.write(";")
            historyFile.write("\n")
            historyFile.write(timestring)
            historyFile.write(";")
            for s in stock.values():
                historyFile.write(str(s))
                historyFile.write(";")
            historyFile.write("\n")