#!/usr/bin/env python3
import re
import urllib.request
from random import choice

userAgents = ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36', 'Opera/9.63 (Macintosh; Intel Mac OS X; U; en) Presto/2.1.1', 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11', 	'Mozilla/5.0 (Windows NT x.y; Win64; x64; rv:10.0) Gecko/20100101 Firefox/10.0', 'Mozilla/5.0 (Macintosh; PPC Mac OS X x.y; rv:10.0) Gecko/20100101 Firefox/10.0', 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0')
categories = ('Lebensmittel_67299','Mutter--Kind_67325','Heim--Garten_67334','Gesundheit--Kosmetik_67391','Elektronik--Technik_67357','Bekleidung--Schmuck_67426','Sport--Freizeit_67399','Fahrzeuge_67414','Dienstleistungen_67502')
offers = []

last = len(offers)
offerPattern = re.compile('<div class=\"design-ad-name\">\n\s+<a href=\"\/de\/anzeige\/(.*)\" title')
quantityPattern = re.compile('marketplaceArticleQuanitityValue\">(\d*)</')

for cat in categories:
    for i in range(1,65535):
        url = "https://markt.kadari.net/de/produkte/" + cat +  "__RUNTIME_DESC_" + str(i) + "_LIST"
        req = urllib.request.Request(
            url, 
            data=None, 
            headers={
            'User-Agent': choice(userAgents)
            }
        )
        f = urllib.request.urlopen(req)
        content = f.read().decode('utf-8')
        offers.extend(offerPattern.findall(content))
        if (len(offers) == last):
            break
        else:
            last = len(offers)

offers = sorted(list(set(offers)))

for offer in offers:
    url = "https://markt.kadari.net/de/anzeige/" + offer
    req = urllib.request.Request(
        url, 
        data=None, 
        headers={
        'User-Agent': choice(userAgents)
        }
    )
    f = urllib.request.urlopen(req)
    content = f.read().decode('utf-8','replace')
    if (int(quantityPattern.search(content).groups()[0]) > 0):
        print(url)