diff --git a/README.md b/README.md index af53fdd..b235bf9 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,9 @@ In order to install all packages follow the steps below: 1. Move to flask-api folder `cd flask-api` 2. For installing virtual environment - `python3 -m pip install --user virtualenv` 3. Create A Virtual env - `python3 -m venv env` -4. Activate virtual env - `source env/bin/activate` +4. Activate virtual env + - For Mac/Linux : `source env/bin/activate` + - For Windows : `.\env\Scripts\activate` 5. `pip3 install -r requirements.txt` 6. `flask run` @@ -100,7 +102,17 @@ The following is a high-level overview of relevant files and folders. ``` DocVerifier/ ├── flask-api/ +│ ├── datasets │ ├── static/uploads +│ ├── model +│ ├── scraper +│ ├── templates +│ ├── .gitignore +│ ├── Procfile +│ ├── nltk.txt +│ ├── requirements.txt +│ ├── runtime.txt +│ ├── output.txt │ └── app.py └── custom_greeting/ @@ -111,18 +123,31 @@ DocVerifier/ │ ├── custom_greeting_assets/ │ │ ├── assets │ │ └── public - ├── dfx.json - ├── package.json - |__ webpack.config.js - |__tsconfig.json - |__ .gitignore + ├── dfx.json + ├── package.json + |__ webpack.config.js + |__ tsconfig.json + |__ canister_ids.json + |__ README.md + |__ package-lock.json + |__ .gitignore | - |__get-current-tab-url + |__chrome-extension |_ background.js |_ icon.png |_ manifest.json |_ window.html + |_ icon.svg |_ style.css +|__images + |_ demo.gif +|__jupyter-notebooks + |_ privacy_policy_predictor.ipynb + |_ web_Scraping.ipynb +|__ .gitignore +|__ CODE_OF_CONDUCT.md +|__ LICENSE +|__ README.md ``` diff --git a/chrome-extension/manifest.json b/chrome-extension/manifest.json index 4f0c698..4c7fe11 100644 --- a/chrome-extension/manifest.json +++ b/chrome-extension/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "DocVerifier!", "description": "This extension verifies any agreement and policies!", -"version": "1.0.0", +"version": "1.0.1", "browser_action": { "default_icon": "icon.png", "default_popup": "window.html" diff --git a/flask-api/app.py b/flask-api/app.py index 83646d1..d2cc8cf 100644 --- a/flask-api/app.py +++ b/flask-api/app.py @@ -50,9 +50,13 @@ def testfn(): # POST request if request.method == "POST": tabUrl = request.get_json()["tabUrl"] - url_list = collect_url_links(tabUrl) # parse as JSON + currentTabUrl = tabUrl.split('/') + url = "https://"+currentTabUrl[2] + logger.info("tab url", url) + + url_list = collect_url_links(url) # parse as JSON # get the current tab url - logger.info(url_list) + logger.info("privacy urls", url_list) for link in url_list: # get the list of privacy policies logger.info(link) diff --git a/flask-api/output.txt b/flask-api/output.txt index 9e53c73..e69de29 100644 --- a/flask-api/output.txt +++ b/flask-api/output.txt @@ -1 +0,0 @@ -All . Getting Set up Git Create a Fork a Be social glossary Git Git and learning Learning about ' s About Advanced Security of Access on about to ’ s up for Sign up for a new account your address Setting up a trial of Enterprise Cloud Setting up a trial of Enterprise Server Exploring early access with feature preview for mobile Keyboard connectivity Exploring on Finding ways to contribute to open source on Saving with Following people Getting with Git Setting your in Git your in Git Why is Git always for my password ? from the Git About remote remote text with Git Git to handle line Git Pushing to a remote repository Getting from a remote repository Dealing with non - fast - forward Splitting a out into a new repository About Git About Git rebase Git rebase on the command line merge after a Git rebase User user account About your personal dashboard your theme your multiple user Converting a user into an organization your user account Permission for a user account repository Permission for user - project the default branch name for your security and analysis for your user account access to your user account ' s project Best for leaving your company What does the ' Available for hire ' do ? an address to your account your primary address Setting a backup address Setting your commit address Blocking command line that expose your personal address your or of marketing from access to your personal Inviting to a personal repository Removing a collaborator from a personal repository Removing yourself from a collaborator ' s repository ownership continuity of your user account ' s your membership in About organization membership an organization people ' s in an organization organization approval for or organization membership your Removing yourself from an organization your profile About your profile About your organization ' s profile your profile your profile Pinning to your profile contribution on your profile on your profile Showing an overview of your activity on your profile or your private on your profile Sending your Enterprise Server to your . profile Why are my not showing up on my profile ? on your Authentication and on Your enterprise Billing and Writing on , , and to your project with and pull your work on Searching for information on your a repository repository data with Supply chain security Secure in a repository large your Extending Finding talent with Working with Support Understanding how and your data Site policy of Service Corporate of Service Privacy Statement Data Protection Agreement ( Non - Enterprise ) Global Privacy and data protection for your organization Additional Additional Product Policy Policy content removal Takedown Policy Guide to a Takedown Notice Guide to a Counter Notice Policy Private Information Removal Policy and Bug Bounty Program Legal Safe Harbor Responsible Disclosure of Security for Legal of User Data Government Takedown Policy Acceptable Use ' s Notice about the Consumer Privacy Act Community Community Forum Code of Conduct Registered Developer Agreement of Service Developer Agreement - release Program Research Program Open Source and and Trade Deceased User Policy Statement Against Modern Slavery and Child Labor Anti - Bribery Statement Candidate Privacy Policy and Entertainment Policy Event Event Code of Conduct Commitment Explore by product . . Enterprise Code security REST Building Education Atom Electron ( Simplified ) ( ) ( ) do ( ) . Site policy Privacy Statement Privacy Statement In this article The short version Summary Privacy Statement What information What information does not collect How your information How we share the information we collect Repository contents Other important information Additional How you can access and control the information we collect Our use of and How your information ' s global privacy How we communicate with you to our Privacy Statement License Effective date : 19 , 2020 Thanks for . (“ ”, “ we ”) with your source code , your , and your personal information . Holding on to your private information is a serious responsibility , and we want you to know how we ' re handling it . All have their definition in ’ s of Service , unless otherwise noted here . The short version We use your personal information as this Privacy Statement . No matter where you are , where you live , or what your citizenship is , we provide the same high standard of privacy protection to all our around the world , regardless of their country of origin or location . Of course , the short version and the Summary below don ' t tell you everything , so please read on for more . Summary can you find there ? What information information directly from you for your registration , payment , , and user profile . We also automatically collect from you your usage information , , and device information , subject , where necessary , to your consent . may also collect User Personal Information from third . We only collect the minimum amount of personal information necessary from you , unless you choose to provide more . What information does not don ’ t knowingly collect information from under 13 , and we don ’ t collect Sensitive Personal Information . How your this section , we describe the ways in which we use your information , to provide you the Service , to communicate with you , for security and compliance , and to improve our Service . We also describe the legal basis upon which we process your information , where legally . How we share the information we may share your information with third under one of the following : with your consent , with our service , for security , to comply with our legal , or when there is a change of control or sale of corporate or business . We do not sell your personal information and we do not host advertising on . You can see a list of the service that access your information . Other important provide additional information specific to repository contents , public information , and on . Additional provide information about additional service , third - party , , and . How you can access and control the information we provide ways for you to access , alter , or delete your personal information . Our use of and only use strictly necessary to provide , secure and improve our service . We offer a page that this very transparent . Please see this section for more information . How your take all reasonably necessary to protect the confidentiality , integrity , and availability of your personal information on and to protect the resilience of our . ' s global privacy provide the same high standard of privacy protection to all our around the world . How we communicate with communicate with you by . You can control the way we contact you in your account , or by us . the unlikely event that we are unable to resolve a privacy concern quickly and thoroughly , we provide a path of dispute resolution . to our Privacy notify you of material to this Privacy Statement 30 days before any such become effective . You may also track in our Site Policy repository . Privacy Statement is licensed under the Creative Commons Zero license . feel free to contact us if you have about our Privacy Statement . provide links to some of the Privacy Statement . Privacy Statement What information " User Personal Information " is any information about one of our which could , alone or together with other information , personally identify them or otherwise be reasonably linked or connected with them . Information such as a and password , an address , a real name , an protocol ( ) address , and a photograph are of “ User Personal Information .” User Personal Information does not include , non - personally information that does not identify a User or cannot otherwise be reasonably linked or connected with them . We may use such , non - personally information for research and to operate , analyze , improve , and optimize our and Service . Information provide directly to Registration information We require some basic information at the time of account creation . When you create your own and password , we ask you for a valid address . Payment information If you sign on to a Account with us , send funds through the Program , or buy an application on , we collect your full name , address , and credit card information or information . Please note , does not process or store your credit card information or information , but our third - party payment processor does . If you list and sell an application on , we require your banking information . If you raise funds through the Program , we require some additional information through the registration process for you to participate in and receive funds through those and for compliance . Profile information You may choose to give us more information for your Account profile , such as your full name , an which may include a photograph , your biography , your location , your company , and a to a third - party . This information may include User Personal Information . Please note that your profile information may be visible to other of our Service . Information automatically from your use of the Service Transactional information If you have a Account with us , sell an application listed on , or raise funds through the Program , we automatically collect certain information about your on the Service , such as the date , time , and amount . Usage information If you ' re our Service or , we automatically collect the same basic information that most collect , subject , where necessary , to your consent . This information about how you use the Service , such as the you view , the site , your address and session information , and the date and time of each request . This is information we collect from every visitor to the , whether they have an Account or not . This information may include User Personal information . As further below , we automatically collect information from ( such as ID and ) to keep you logged in , to remember your , to identify you and your device and to analyze your use of our service . Device information We may collect certain information about your device , such as its address , browser or client application information , language preference , operating system and application version , device type and ID , and device model and manufacturer . This information may include User Personal information . Information we collect from third may collect User Personal Information from third . For example , this may happen if you sign up for training or to receive information about from one of our , , or . does not purchase User Personal Information from third - party data . What information does not collect We do not intentionally collect “ Sensitive Personal Information ”, such as personal data revealing racial or ethnic origin , political , religious or philosophical , or trade union membership , and the of genetic data , biometric data for the purpose of uniquely a natural person , data concerning health or data concerning a natural person ’ s sex life or sexual orientation . If you choose to store any Sensitive Personal Information on our , you are responsible for with any regulatory regarding that data . If you are a child under the age of 13 , you may not have an Account on . does not knowingly collect information from or direct any of our content specifically to under 13 . If we learn or have reason to suspect that you are a User who is under the age of 13 , we will have to close your Account . We don ' t want to discourage you from learning to code , but those are the . Please see our of Service for information about Account termination . Different may have different minimum age , and if you are below the minimum age for providing consent for data collection in your country , you may not have an Account on . We do not intentionally collect User Personal Information that is in your or other free - form content . Any personal information within a user ' s repository is the responsibility of the repository owner . How your information We may use your information for the following : We use your Registration Information to create your account , and to provide you the Service . We use your Payment Information to provide you with the Account service , the service , the Program , or any other service you request . We use your User Personal Information , specifically your , to identify you on . We use your Profile Information to fill out your Account profile and to share that profile with other if you ask us to . We use your address to communicate with you , if you ' said that ' s , and only for the you ’ said that ’ s . Please see our section on communication for more information . We use User Personal Information to respond to support . We use User Personal Information and other data to make for you , such as to suggest you may want to follow or contribute to . We learn from your public behavior on — such as the you star — to determine your , and we recommend similar . These are , but they have no legal impact on your . We may use User Personal Information to invite you to take part in , beta , or other research , subject , where necessary , to your consent . We use Usage Information and Device Information to better understand how our use and to improve our and Service . We may use your User Personal Information if it is necessary for security or to investigate possible fraud or to harm or our . We may use your User Personal Information to comply with our legal , protect our intellectual property , and enforce our of Service . We limit our use of your User Personal Information to the listed in this Privacy Statement . If we need to use your User Personal Information for other , we will ask your permission first . You can always see what information we have , how we ' re it , and what you have given us in your user profile . Our legal bases for information To the extent that our of your User Personal Information is subject to certain international ( , but not limited to , the Union ' s General Data Protection Regulation ( )), is to notify you about the legal basis on which we process User Personal Information . User Personal Information on the following legal bases : Contract Performance : When you create a Account , you provide your Registration Information . We require this information for you to enter into the of Service agreement with us , and we process that information on the basis of that contract . We also process your and address on other legal bases , as below . If you have a Account with us , we collect and process additional Payment Information on the basis of that contract . When you buy or sell an application listed on our or , when you send or receive funds through the Program , we process Payment Information and additional in order to perform the contract that to those . Consent : We rely on your consent to use your User Personal Information under the following : when you fill out the information in your user profile ; when you decide to participate in a training , research project , beta program , or survey ; and for marketing , where applicable . All of this User Personal Information is entirely optional , and you have the ability to access , modify , and delete it at any time . While you are not able to delete your address entirely , you can make it private . You may withdraw your consent at any time . Legitimate : Generally , the remainder of the of User Personal Information we perform is necessary for the of our legitimate interest , for example , for legal compliance , security , or to maintain ongoing confidentiality , integrity , availability , and resilience of ’ s , , and Service . If you would like to request deletion of data we process on the basis of consent or if you object to our of personal information , please use our Privacy contact form . How we share the information we collect We may share your User Personal Information with third under one of the following : With your consent We share your User Personal Information , if you consent , after you know what information will be , with whom , and why . For example , if you purchase an application listed on our , we share your to allow the application Developer to provide you with . Additionally , you may direct us through your on to share your User Personal Information . For example , if you join an Organization , you indicate your willingness to provide the owner of the Organization with the ability to view your activity in the Organization ’ s access log . With service We share User Personal Information with a limited number of service who process it on our behalf to provide or improve our Service , and who have agreed to privacy similar to the in our Privacy Statement by data protection or making similar . Our service perform payment , customer support ticketing , network data transmission , security , and other similar . While all User Personal Information in the United , our service may process data outside of the United or the Union . If you would like to know who our service are , please see our page on . For security If you are a member of an Organization , may share your , Usage Information , and Device Information associated with that Organization with an owner and / or administrator of the Organization , to the extent that such information is provided only to investigate or respond to a security incident that or the security of that particular Organization . For legal disclosure for transparency in with legal process and legal . Unless from doing so by law or court order , or in rare , exigent , we make a reasonable effort to notify of any legally or disclosure of their information . may disclose User Personal Information or other information we collect about you to law enforcement if in response to a valid subpoena , court order , search warrant , a similar government order , or when we believe in good faith that disclosure is necessary to comply with our legal , to protect our property or , or those of third or the public at large . For more information about our disclosure in response to legal , see our for Legal of User Data . Change in control or sale We may share User Personal Information if we are involved in a merger , sale , or acquisition of corporate or business . If any such change of ownership , we will ensure that it is under that preserve the confidentiality of User Personal Information , and we will notify you on our or by before any transfer of your User Personal Information . The organization any User Personal Information will have to honor any we made in our Privacy Statement or of Service . Aggregate , non - personally information We share certain , non - personally information with about how our , collectively , use , or how our respond to our other , such as our or . We do not sell your User Personal Information for monetary or other consideration . Please note : The Consumer Privacy Act of 2018 (“ ”) to state in their privacy policy whether or not they disclose personal information in exchange for monetary or other valuable consideration . While only , we voluntarily extend its core for people to control their data to all of our , not just those who live in . You can learn more about the and how we comply with it here . Repository contents Access to private If your repository is private , you control the access to your Content . If you include User Personal Information or Sensitive Personal Information , that information may only be accessible to in accordance with this Privacy Statement . personnel do not access private repository content except for security to assist the repository owner with a support matter to maintain the integrity of the Service to comply with our legal if we have reason to believe the contents are in violation of the law , or with your consent . However , while we do not generally search for content in your , we may scan our and content to detect certain or security , known active , known in , or other content known to violate our of Service , such as violent extremist or terrorist content or child exploitation imagery , based on algorithmic fingerprinting ( collectively , " scanning "). Our of Service more on private . Please note , you may choose to disable certain access to your private that is by default as part of providing you with the Service ( for example , scanning to enable Dependency Graph and ). will provide notice regarding our access to private repository content , unless for legal disclosure , to comply with our legal , or where otherwise bound by under law , for scanning , or if in response to a security threat or other risk to security . Public If your repository is public , anyone may view its contents . If you include User Personal Information , Sensitive Personal Information , or confidential information , such as or , in your public repository , that information may be indexed by search or used by third . Please see more about User Personal Information in public . Other important information Public information on Many of and are public - facing . If your content is public - facing , third may access and use it in compliance with our of Service , such as by your profile or or data via our . We do not sell that content ; it is yours . However , we do allow third , such as research or , to compile public - facing information . Other third , such as data , have been known to scrape and compile data as well . Your User Personal Information associated with your content could be by third in these of data . If you do not want your User Personal Information to appear in third ’ of data , please do not make your User Personal Information publicly available and be sure to configure your address to be private in your user profile and in your git commit . We currently set ' address to private by default , but legacy may need to update their . If you would like to compile data , you must comply with our of Service regarding information usage and privacy , and you may only use any public - facing User Personal Information you gather for the purpose for which our user authorized it . For example , where a user made an address public - facing for the purpose of identification and attribution , do not use that address for the of sending unsolicited to or selling User Personal Information , such as to , , and job , or for commercial advertising . We expect you to reasonably secure any User Personal Information you have from , and to respond promptly to , removal , and " do not contact " from or . Similarly , on may include publicly available User Personal Information collected as part of the collaborative process . If you have a complaint about any User Personal Information on , please see our section on . You may indicate , through your on , that you are willing to share your User Personal Information . If you collaborate on or become a member of an Organization , then its Account may receive your User Personal Information . When you accept an invitation to an Organization , you will be notified of the of information may be able to see ( for more information , see About Organization Membership ). If you accept an invitation to an Organization with a domain , then the of that Organization will be able to see your full address ( es ) within that Organization ' s domain ( s ). Please note , may share your , Usage Information , and Device Information with the owner ( s ) of the Organization you are a member of , to the extent that your User Personal Information is provided only to investigate or respond to a security incident that or the security of that particular Organization . If you collaborate on or become a member of an Account that agreed to the Corporate of Service and a Data Protection Addendum ( ) to this Privacy Statement , then that in the event of any between this Privacy Statement and the with respect to your activity in the Account . Please contact the Account for more information about how they might process your User Personal Information in their Organization and the ways for you to access , update , alter , or delete the User Personal Information in the Account . Additional Third party You have the option of or third - party , known as " Developer ," to your Account . These Developer are not necessary for your use of . We will share your User Personal Information with third when you ask us to , such as by a Developer Product from the ; however , you are responsible for your use of the third - party Developer Product and for the amount of User Personal Information you choose to share with it . You can check our documentation to see what information is provided when you authenticate into a Developer Product your profile . If you create a , it is your responsibility to post a privacy statement that accurately how you collect , use , and share personal information and other visitor information , and how you comply with applicable data privacy , , and . Please note that may collect User Personal Information from to your , of visitor , to comply with legal , and to maintain the security and integrity of the and the Service . You can also add from , such as our , our Atom application , or other application and account , to your Account . These each have their own and may collect different of User Personal Information ; however , all are subject to this Privacy Statement , and we collect the amount of User Personal Information necessary , and use it only for the purpose for which you have given it to us . How you can access and control the information we collect If you ' re already a user , you may access , update , alter , or delete your basic user profile information by your user profile or Support or Premium Support . You can control the information we collect about you by limiting what information is in your profile , by keeping your information current , or by Support or Premium Support . If information about you , such as information from third , and you do not have an account , then you may , subject to applicable law , access , update , alter , delete , or object to the of your personal information by Support or Premium Support . Data portability As a User , you can always take your data with you . You can clone your to your , for example , or you can use our Data Portability to information we have about you . Data retention and deletion of data Generally , User Personal Information for as long as your account is active or as to provide you . If you would like to cancel your account or delete your User Personal Information , you may do so in your user profile . We retain and use your information as necessary to comply with our legal , resolve , and enforce our , but barring legal , we will delete your full profile ( within reason ) within 90 days of your request . You may contact Support or Premium Support to request the erasure of the data we process on the basis of consent within 30 days . After an account been , certain data , such as to other ' and in ' , will remain . However , we will delete or de - identify your User Personal Information , your and address , from the author field of , pull , and by them with a ghost user . That said , the address you have via your Git commit will always be associated with your in the Git system . If you choose to make your address private , you should also update your Git commit . We are unable to change or delete data in the Git commit history — the Git is designed to maintain a record — but we do enable you to control what information you put in that record . Our use of and only strictly necessary . are small text that often store on computer hard or mobile of . We use solely to provide , secure , and improve our service . For example , we use them to keep you logged in , remember your , identify your device for security , analyze your use of our service , compile statistical , and provide information for future development of . We use our own for analytics , but do not use any third - party analytics service . By our service , you agree that we can place these of on your computer or device . If you disable your browser or device ’ s ability to accept these , you will not be able to log in or use our service . We provide more information about on on our and page that the we set , the needs we have for those , and the expiration of such . " Do Not Track " ( ) is a privacy preference you can set in your browser if you do not want to collect and share certain of information about your activity from third party . to browser and the W3C standard for to . If you would like to set your browser to signal that you would not like to be tracked , please check your browser ' s documentation for how to enable that signal . There are also good that block , such as Privacy Badger . How your information all reasonably necessary to protect User Personal Information from unauthorized access , alteration , or destruction ; maintain data accuracy ; and help ensure the appropriate use of User Personal Information . a written security information program . Our program : with industry ; security reasonably designed to protect the confidentiality , integrity , availability , and resilience of our ' data ; is appropriate to the nature , size , and complexity of ’ s business ; incident response and data breach notification ; and with applicable information security - related and in the geographic where does business . In the event of a data breach that your User Personal Information , we will act promptly to mitigate the impact of a breach and notify any affected without undue delay . Transmission of data on is , ( ), and git repository content is at rest . We manage our own and at top - tier data with high level of physical and network security , and when data is with a third - party storage provider , it is . No method of transmission , or method of electronic storage , is 100 % secure . Therefore , we cannot guarantee its absolute security . For more information , see our security . ' s global privacy , . and , for those in the Economic Area , the United Kingdom , and , B . V . are the responsible for the of your personal information in connection with the Service , except ( a ) with respect to personal information that was added to a repository by its , in which case the owner of that repository is the controller and is the processor ( or , if the owner as a processor , will be the ); or ( b ) when you and have into a separate agreement that data privacy ( such as a Data Agreement ). Our are : , ., 88 Colin P . Kelly . Street , San , CA 94107 . B . V ., 68 - 72 , 1017 , The . We store and process the information that we collect in the United in accordance with this Privacy Statement , though our service may store and process data outside the United . However , we understand that we have from different and with different privacy , and we try to meet those needs even when the United does not have the same privacy framework as other . We provide the same high standard of privacy protection — as in this Privacy Statement — to all our around the world , regardless of their country of origin or location , and we are of the of notice , choice , accountability , security , data integrity , access , and recourse we provide . We work hard to comply with the applicable data privacy wherever we do business , working with our Data Protection Officer as part of a cross - functional team that our privacy compliance . Additionally , if our or have access to User Personal Information , they must sign that require them to comply with our privacy and with applicable data privacy . In particular : clear of unambiguous , informed , specific , and freely given consent at the time of data collection , when we collect your User Personal Information consent as a basis . We collect only the minimum amount of User Personal Information necessary for our , unless you choose to provide more . We encourage you to only give us the amount of data you are comfortable . We offer you simple of , , or the User Personal Information we have collected , where legally permitted . We provide our notice , choice , accountability , security , and access regarding their User Personal Information , and we limit the purpose for it . We also provide our a method of recourse and enforcement . Cross - border data personal information both inside and outside of the United and on Standard Contractual as the legally provided mechanism to lawfully transfer data from the Economic Area , the United Kingdom , and to the United . In addition , is certified to the EU - US and Swiss - US Privacy Shield . To learn more about our cross - border data , see our Global Privacy . How we communicate with you We use your address to communicate with you , if you ' said that ' s , and only for the you ’ said that ’ s . For example , if you contact our Support team with a request , we respond to you via . You have a lot of control over how your address is used and on and through . You may manage your communication in your user profile . By design , the Git version control system many with a User ' s address , such as commit . We are not able to change many of the Git system . If you would like your address to remain private , even when you ’ re on public , you can create a private address in your user profile . You should also update your local Git configuration to use your private address . This will not change how we contact you , but it will affect how see you . We set current ' address private by default , but legacy may need to update their . Please see more about in commit here . Depending on your , may occasionally send notification about in a repository you ’ re watching , new , for feedback , important policy , or to offer customer support . We also send marketing , based on your and in accordance with applicable and . There ' s an “ ” link at the bottom of each of the marketing we send you . Please note that you cannot opt out of important from us , such as from our Support team or system , but you can configure your in your profile to opt out of other . Our may contain a tag , which is a small , clear image that can tell us whether or not you have an and what your address is . We use this tag to make our more effective for you and to make sure we ’ re not sending you unwanted . If you have about the way is handling your User Personal Information , please let us know immediately . We want to help . You may contact us by filling out the Privacy contact form . You may also us directly at privacy @ . with the subject line " Privacy ." We will respond promptly — within 45 days at the latest . You may also contact our Data Protection Officer directly . Our United EU Data Protection BV88 Colin P . Kelly . St . 68 - 72San , CA 941071017 @ . @ . Dispute resolution process In the unlikely event that a dispute between you and regarding our handling of your User Personal Information , we will do our best to resolve it . Additionally , if you are a resident of an EU member state , you have the right to file a complaint with your local supervisory authority , and you might have more . to our Privacy Statement Although most are likely to be minor , may change our Privacy Statement from time to time . We will provide notification to of material to this Privacy Statement through our at least 30 days prior to the change taking effect by posting a notice on our home page or sending to the primary address in your account . We will also update our Site Policy repository , which all to this policy . For other to this Privacy Statement , we encourage to watch or to check our Site Policy repository frequently . License This Privacy Statement is licensed under this Creative Commons Zero license . For , see our site - policy repository . regarding ' s Privacy Statement or information should be directed to our Privacy contact form . Below are of this document into other . In the event of any conflict , uncertainty , or apparent inconsistency between any of those and the version , this version is the version . pour la version : de de Other For of this statement into other , please visit :// . . / and select a language from the drop - down menu under “ .” Did this doc help you ? Privacy policy Want to learn about new and ? Sign up for ! We ' re continually improving our . We ' d love to hear how we can do better . Let us know what we can do better Optional Can we contact you if we have more ? Optional Send Thank you ! Your feedback been . Help us make these great ! All are open source . See something that ' s wrong or unclear ? Submit a pull request . Make a contribution Or , learn how to contribute . Still need help ? Ask the community Contact support © 2021 , . Privacy Security Status Help Contact Developer Training About diff --git a/flask-api/scraper/getPolicyText.py b/flask-api/scraper/getPolicyText.py index 02a9b53..59a642d 100644 --- a/flask-api/scraper/getPolicyText.py +++ b/flask-api/scraper/getPolicyText.py @@ -9,6 +9,7 @@ def getPolicies(url_link): + source = requests.get(url_link).text soup = BeautifulSoup(source, "lxml") policies = soup.find("body") diff --git a/flask-api/scraper/getUrls.py b/flask-api/scraper/getUrls.py index 9bd7ede..fe6e90d 100644 --- a/flask-api/scraper/getUrls.py +++ b/flask-api/scraper/getUrls.py @@ -1,20 +1,28 @@ from bs4 import BeautifulSoup import requests import re +import time +import random # function to get all the policy urls from a website def collect_url_links(url_link) -> list: + + url_list = [] pattern = re.compile(r'^http') source = requests.get(url_link).text + # List for Randomizing our request rate + rate = [i/10 for i in range(10)] soup = BeautifulSoup(source, 'lxml') a_tag = soup.find_all("a") # Gives you the list of all the a tags for i in a_tag: if i.text in ["Privacy", "Terms", "Privacy Policy", "Terms of Service"]: url = i["href"] url_list.append(url) + # Randomizing our request rate + time.sleep(random.choice(rate)) for i in range(len(url_list)): matches = pattern.finditer(url_list[i])