import requests
from bs4 import BeautifulSoup
import pandas as pd
from tqdm import tqdm_notebook as tqdm
import csv

def get_pages(main):
    try:
        soup = connect(main)
        n_pages = [_.get_text(strip=True) for _ in soup.find('ul', {'class': 'pagination pagination__number'}).find_all('li')]
        last_page = int(n_pages[-1])
        pages = [main]
        
        for n in range(2,last_page+1):    
            page_num = "/?pag={}".format(n)
            pages.append(main + page_num)
    except:
        pages = [main]
        
    return pages

def connect(web_addr):
    resp = requests.get(web_addr)
    return BeautifulSoup(resp.content, "html.parser")

def get_areas(website):
    data = connect(website)
    areas = []
    for ultag in data.find_all('ul', {'class': 'breadcrumb-list breadcrumb-list_list breadcrumb-list__related'}):
        for litag in ultag.find_all('li'):
            for i in range(len(litag.text.split(','))):
                areas.append(litag.text.split(',')[i])
    areas = [x.strip() for x in areas]
    urls = []
    
    for area in areas:
        url = website + '/' + area.replace(' ','-').lower()
        urls.append(url)
    
    return areas, urls

def get_apartment_links(website):
    data = connect(website)
    links = []
    for link in data.find_all('ul', {'class': 'annunci-list'}):
        for litag in link.find_all('li'):
            try:
                links.append(litag.a.get('href'))
            except:
                continue
    return links

def scrape_link(website):
    data = connect(website)
    info = data.find_all('dl', {'class': 'im-features__list'})
    comp_info = pd.DataFrame()
    cleaned_id_text = []
    cleaned_id__attrb_text = []
    for n in range(len(info)):
        for i in info[n].find_all('dt'):
            cleaned_id_text.append(i.text)
        for i in info[n].find_all('dd'):
            cleaned_id__attrb_text.append(i.text)
    comp_info['Id'] = cleaned_id_text
    comp_info['Attribute'] = cleaned_id__attrb_text
    comp_info
    feature = []
    for item in comp_info['Attribute']:
        try:
            feature.append(clear_df(item))
        except:
            feature.append(ultra_clear_df(item))
    comp_info['Attribute'] = feature
    return comp_info['Id'].values, comp_info['Attribute'].values

def remove_duplicates(x):
    return list(dict.fromkeys(x))

def clear_df(the_list):
    the_list = (the_list.split('\n')[1].split('  '))
    the_list = [value for value in the_list if value != ''][0]
    return the_list

def ultra_clear_df(the_list):
    the_list = (the_list.split('\n\n')[1].split('  '))
    the_list = [value for value in the_list if value != ''][0]
    the_list = (the_list.split('\n')[0])
    return the_list

# Получаем районы внутри города (районы)
website = "https://www.immobiliare.it/affitto-case/torino"
districts = get_areas(website)
print("Вот ссылки на районы **\n**")
print(districts)

# Теперь нам нужно найти все ссылки на объявления, чтобы по одному спарсить информацию внутри них
address = []
location = []
try:
    for url in tqdm(districts):
        pages = get_pages(url)
        for page in pages:
            add = get_apartment_links(page)
            address.append(add)
            for num in range(0,len(add)):
                location.append(url.rsplit('/', 1)[-1])
except Exception as e:
    print(e)
    continue

announces_links = [item for value in address for item in value]

# Проверяем, имеет ли это смысл, и сохраняем
print("Количество объявлений:**\n**")
print(len(announces_links))

with open('announces_list.csv', 'w') as myfile:
    wr = csv.writer(myfile)
    wr.writerow(announces_links)

## Теперь мы передаем все ссылки на объявления в функцию scrape_link, чтобы получить информацию о квартирахdf_scrape = pd.DataFrame()
to_be_dropped = []
counter = 0
for link in tqdm(list(announces_links)):
    counter=counter+1
    try:
        names, values = scrape_link(link)
        temp_df = pd.DataFrame(columns=names)
        temp_df.loc[len(temp_df), :] = values[0:len(names)]
        df_scrape = df_scrape.append(temp_df, sort=False)
    except Exception as e:
        print(e)
        to_be_dropped.append(counter)
        print(to_be_dropped)
        continue## В конечном итоге сохраняем полезную информацию, полученную в процессе парсингаpd.DataFrame(location).to_csv('location.csv', sep=';')
pd.DataFrame(to_be_dropped).to_csv('to_be_dropped.csv', sep=';')

print(df_scrape.shape)
df_scrape[‘district’] = location
df_scrape[‘links’] = announces_links
df_scrape.columns = map(str.lower, df_scrape.columns)
df_scrape.to_csv(‘dataset.csv’, sep=”;”)

df_scrape = df_scrape[['contratto', 'zona', 'tipologia', 'superficie', 'locali', 'piano', 'tipo proprietà', 'prezzo', 'spese condominio', 'spese aggiuntive', 'anno di costruzione', 'stato', 'riscaldamento', 'climatizzazione', 'posti auto', 'links']]def cleanup(df):
    price = []
    rooms = []
    surface = []
    bathrooms = []
    floor = []
    contract = []
    tipo = []
    condominio = []
    heating = []
    built_in = []
    state = []
    riscaldamento = []
    cooling = []
    energy_class = []
    tipologia = []
    pr_type = []
    arredato = []
    
    for tipo in df['tipologia']:
        try:
            tipologia.append(tipo)
        except:
            tipologia.append(None)
    
    for superficie in df['superficie']:
        try:
            if "м" in superficie:
                #z = superficie.split('|')[0]
                s = superficie.replace(" м²", "")
                surface.append(s)
        except:
            surface.append(None)
    
    for locali in df['locali']:
        try:
            rooms.append(locali[0:1])
        except:
            rooms.append(None)
    
    for prezzo in df['prezzo']:
        try:
            price.append(prezzo.replace("Аренда ", "").replace("€ ", "").replace("/месяц", "").replace(".",""))
        except:
            price.append(None)
            
    for contratto in df['contratto']:
        try:
            contract.append(contratto.replace("\n ",""))
        except:
            contract.append(None)
    
    for piano in df['piano']:
        try:
            floor.append(piano.split(' ')[0])
        except:
            floor.append(None)
    
    for tipologia in df['tipo proprietà']:
        try:
            pr_type.append(tipologia.split(',')[0])
        except:
            pr_type.append(None)
            
    for condo in df['spese condominio']:
        try:
            if "месяц" in condo:
                condominio.append(condo.replace("€ ","").replace("/месяц",""))
            else:
                condominio.append(None)
        except:
            condominio.append(None)
        
    for ii in df['spese aggiuntive']:
        try:
            if "год" in ii:
                mese = int(int(ii.replace("€ ","").replace("/год","").replace(".",""))/12)
                heating.append(mese)
            else:
                heating.append(None)
        except:
            heating.append(None)
   
    for anno_costruzione in df['anno di costruzione']:
        try:
            built_in.append(anno_costruzione)
        except:
            built_in.append(None)
    
    for stato in df['stato']:
        try:
            stat = stato.replace(" ","").lower()
            state.append(stat)
        except:
            state.append(None)
    
    for tipo_riscaldamento in df['riscaldamento']:
        try:
            if 'Централизованное' in tipo_riscaldamento:
                riscaldamento.append('централизованное')
            elif 'Автономное' in tipo_riscaldamento:
                riscaldamento.append('автономное')
        except:
            riscaldamento.append(None)
    
    for clima in df['climatizzazione']:
        try:
            cooling.append(clima.lower().split(',')[0])
        except:
            cooling.append('None')
    
    final_df = pd.DataFrame(columns=['contract', 'district', 'renting_type', 'surface', 'locals', 'floor', 'property_type', 'price', 'spese condominio', 'other_expences', 'building_year', 'status', 'heating', 'air_conditioning', 'energy_certificate', 'parking_slots'])#, 'Arredato S/N'])
    final_df['contract'] = contract
    final_df['renting_type'] = tipologia
    final_df['surface'] = surface
    final_df['locals'] = rooms
    final_df['floor'] = floor
    final_df['property_type'] = pr_type
    final_df['price'] = price
    final_df['spese condominio'] = condominio
    final_df['heating_expences'] = heating
    final_df['building_year'] = built_in
    final_df['status'] = state
    final_df['heating_system'] = riscaldamento
    final_df['air_conditioning'] = cooling
    #final_df['classe energetica'] = energy_class
    final_df['district'] = df['zona'].values
    #inal_df['Arredato S/N'] = arredato
    final_df['announce_link'] = announces_links
    
    return final_dffinal = cleanup(df_scrape)
final.to_csv('regression_dataset.csv', sep=";")