from bs4 import BeautifulSoup
import urllib.request
import requests
import re
from selenium import webdriver
import time


def main():
    headers = {'User-Agent': 'chromium'}
    url = "https://rosreestr.ru/wps/portal/p/cc_ib_portal_services/cc_ib_ais_fdgko/!ut/p/z1/lZBLb8IwEIR_Cweu2TWvoN5MhKIEIR4VbeJLZJBJIpzYcgyIf4-j9tDSooi9zeqbndECgwRYzS9lzm2pai6dTtkkCzejOQlGZBHuZhOk0TqefUQBIvrw-QBsiO8AuiBDEiKuCLDX_H-A1o9PhqLzs4eI3w2CQQfQVuwKSV1J_2nJKcJ7e-OgamuUlMJAKsvGboVWxjYQA8ul2n99k9b74TQHZsRRGGG8s3HrwlrdvPWxj_zKb97l5B1U9S10of_zFaqxkPzAQVe7BMt1VWVLO05p7w7mrlu-/p0/IZ7_GQ4E1C41KGUB60AIPJBVIC0080=CZ6_GQ4E1C41KGUB60AIPJBVIC0007=MEcontroller!listReports==/?restoreSessionState=true"


    r = requests.get(
        url,
        headers=headers)

    soup = BeautifulSoup(r.text, features="html.parser")
    clean = re.compile('<.*?>')
    tables = soup.find_all("div", attrs={"class": "portlet_content"})

    with open("1.txt", "a", encoding="utf-8") as file:
        for table in tables:
            # print(table)
            rows = table.find_all("td", attrs={"class": "table_row_align_left"})
            for i in rows:
                k = i.find_all("a")
                for u in k:
                    rowsing = u.get('href')
                    k = requests.get(url + rowsing, headers=headers)
                    # print(k)
                    soup1 = BeautifulSoup(k.text, features="html.parser")
                    # time.sleep(5)
                    tak0 = soup1.find_all("div", attrs={"id": "procedure_region_code"})
                    for o in tak0:
                        print(o)
                        row_right = o.find_all("div", attrs={"class": "row_right"})
                        for j in row_right:
                            string = "\t".join([str(x).strip() for x in j.contents])
                            string = re.sub(clean, "\t", string)
                            file.write(string)
                            file.write("\n")
                            print(string)

                    tak = soup1.find_all("div", attrs={"id": "report_executor_name"})
                    for o in tak:
                        row_right = o.find_all("div", attrs={"class": "row_right"})
                        for j in row_right:
                            string = "\t".join([str(x).strip() for x in j.contents])
                            string = re.sub(clean, "\t", string)
                            file.write(string)
                            file.write("\n")
                            print(string)

                    tak1 = soup1.find_all("div", attrs={"id": "report_exec_jur_orgn"})
                    for o in tak1:
                        row_right = o.find_all("div", attrs={"class": "row_right"})
                        for j in row_right:
                            string = "\t".join([str(x).strip() for x in j.contents])
                            string = re.sub(clean, "\t", string)
                            file.write(string)
                            file.write("\n")
                            print(string)

                    tak2 = soup1.find_all("div", attrs={"id": "report_exec_jur_address"})
                    for o in tak2:
                        row_right = o.find_all("div", attrs={"class": "row_right"})
                        for j in row_right:
                            string = "\t".join([str(x).strip() for x in j.contents])
                            string = re.sub(clean, "\t", string)
                            file.write(string)
                            file.write("\n")
                            print(string)


if __name__ == "__main__":
    main()