complete article text and metadata extraction

.gitignore

@@ -2,3 +2,4 @@
 /dist/
 /*.egg
 /*.egg-info
+venv

CLI.md

@@ -0,0 +1,46 @@
+# gnews
+
+CLI to use gnewsclient
+
+How to use?
+-----------
+
+To use, it is recommended to make `virtualenv` and then install all required packages:
+
+* Installing virtualenv:  
+```
+$ sudo pip install virtualenv
+```  
+* Making virtualenv:  
+```
+$ virtualenv venv
+```  
+* Go to your gnewsclient dir and activate it:   
+```
+$ . venv/bin/activate
+```  
+* To install all required packages:  
+ ```
+ $ pip install --editable .
+ or
+ $ sudo pip install --editable .
+```
+
+
+## Usage: `$ gnews [OPTIONS]`
+
+```
+Options:
+  --config         shows default config
+  --query TEXT     shows news about query given
+  --edition TEXT   shows news of edition given, default=United States
+                   (English)
+  --topic TEXT     shows topic given, default=top stories
+  --location TEXT  shows news from location given
+  --language TEXT  shows news in language given, default is english
+  --sheditions     shows list of available editions
+  --shtopics       shows list of available topics
+  --shlangs        shows list of available languages
+  --help           Show this message and exit.
+
+ ```

README.md

@@ -1,4 +1,4 @@
-[![PyPI](https://img.shields.io/badge/PyPi-v1.0.2-f39f37.svg)](https://pypi.python.org/pypi/gnewsclient)
+[![PyPI](https://img.shields.io/badge/PyPi-v1.1.0-f39f37.svg)](https://pypi.python.org/pypi/gnewsclient)
 [![license](https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000)](https://github.com/nikhilkumarsingh/gnewsclient/blob/master/LICENSE.txt)
 
 # gnewsclient
@@ -14,6 +14,8 @@ To install gnewsclient, simply,
 $ pip install gnewsclient
 ```
 
+To install and use **gnewsclient CLI**, follow instructions here [CLI](CLI.md)
+
 ## Filters
 
 Google News feeds use 3 basic filters:

README.rst

@@ -17,6 +17,9 @@ To install gnewsclient, simply,
 
     $ pip install gnewsclient
 
+To install and use **gnewsclient CLI**, follow instructions here
+`CLI <CLI.md>`__
+
 Filters
 -------
 
@@ -152,7 +155,7 @@ Usage
        'estonian', 'indonesian', 'slovenian', 'italian', 'maltese', 'haitian creole', 'esperanto', 'ukrainian', 
        'afrikaans', 'filipino', 'gujarati', 'hebrew', 'telugu', 'greek', 'persian', 'romanian']
 
-.. |PyPI| image:: https://img.shields.io/badge/PyPi-v1.0.2-f39f37.svg
+.. |PyPI| image:: https://img.shields.io/badge/PyPi-v1.1.0-f39f37.svg
    :target: https://pypi.python.org/pypi/gnewsclient
 .. |license| image:: https://img.shields.io/github/license/mashape/apistatus.svg?maxAge=2592000
    :target: https://github.com/nikhilkumarsingh/gnewsclient/blob/master/LICENSE.txt

gnewsclient/gnewsclient.py

@@ -2,7 +2,8 @@
 import requests
 from bs4 import BeautifulSoup
 from .utils import editionMap, topicMap, langMap
-
+from newspaper import Article
+from .userexception import NotFound
 
 class gnewsclient:
 
@@ -41,10 +42,20 @@ def get_config(self):
             'edition': self.edition,
             'topic': self.topic,
             'language': self.language,
-            'loaction': self.location,
+            'location': self.location,
             'query': self.query
         }
         return config
+
+    def reset(self):
+        '''
+        function to reset the parameters
+        '''
+        self.edition = 'United States (English)'
+        self.language = 'english'
+        self.location = None
+        self.query = None
+        self.topic = 'top stories'
 
 
     def get_news(self):
@@ -58,8 +69,14 @@ def get_news(self):
 
         soup = self.load_feed()
         articles = self.scrape_feed(soup)
-        return articles
-
+        object_list = []
+        for a in articles:
+            article = Articledata(a['link'], title=a['title'])
+            object_list.append(article)
+        return object_list
+
+
+
 
     def set_params(self):
         '''
@@ -124,10 +141,30 @@ def scrape_feed(self, soup):
             article = {}
             article['title'] = entry.title.text
             article['link'] = entry.link['href'].split('&url=')[1]
-            try:
-                article['img'] = "https:" + entry.content.text.split('src=\"')[1].split('\"')[0]
-            except:
-                article['img'] = None
-                pass
             articles.append(article)
-        return articles       
+        try:
+            if len(articles)==0:
+                raise NotFound
+        except NotFound:
+                print("The articles for the given response are not found.")
+                return
+        return articles       
+
+
+class Articledata(Article):
+
+    def get_fulltext(self):
+        if self.html=='':
+            self.build()
+        return self.text
+
+    def get_metadata(self):
+        if self.html=='':
+            self.build()
+        return self.meta_data
+    def get_summary(self):
+        if self.html=='':
+            self.build()
+        return self.summary
+
+

gnewsclient/scripts/gnews.py

@@ -0,0 +1,56 @@
+import click
+from gnewsclient import gnewsclient
+
+client = gnewsclient()
+
+@click.command()
+@click.option("--config",is_flag=True,help="shows default config")
+
+@click.option("--query",default=None,help="shows news about query given")
+@click.option("--edition",default="United States (English)",help="shows news of edition given, default=United States (English)")
+@click.option("--topic",default="top stories",help="shows topic given, default=top stories")
+@click.option("--location",default=None,help="shows news from location given")
+@click.option("--language",default="english",help="shows news in language given, default is english")
+
+@click.option("--sheditions",is_flag=True,help="shows list of available editions")
+@click.option("--shtopics",is_flag=True,help="shows list of available topics")
+@click.option("--shlangs",is_flag=True,help="shows list of available languages")
+
+def cli(config,query,edition,topic,location,language,shlangs,shtopics,sheditions):
+	""" CLI to get news """
+
+	client.query = query
+	client.edition = edition
+	client.topic = topic
+	client.location = location
+	client.language = language
+
+	if config:
+		conf = client.get_config()
+		click.echo("The default configuration : ")
+		for keys,value in conf.items():
+			click.echo(str(keys)+" : "+str(value))
+
+	elif shlangs:
+		langs = client.languages
+		click.echo("The languages supported : ")
+		for l in langs:
+			click.echo(l)
+
+	elif sheditions:
+		editions = client.editions
+		click.echo("The editions available : ")
+		for e in editions:
+			click.echo(e)
+
+	elif shtopics:
+		tps = client.topics
+		click.echo("The topics available : ")
+		for t in tps:
+			click.echo(t)
+	else:
+		neews = client.get_news()
+		for n in neews:
+			content = "{}\n{}".format(n['title'],n['link'])
+			click.echo(content)
+			click.echo("\n")

gnewsclient/userexception.py

@@ -0,0 +1,4 @@
+#User defined exceptions for Gnewsclient
+class NotFound(Exception):
+    """Raised when the list articles in the function scapefeed() is empty"""
+    pass

requirements.txt

@@ -0,0 +1,3 @@
+bs4==0.0.1
+html5lib==0.999999999
+requests==2.18.4

setup.py

@@ -9,7 +9,7 @@ def readme():
 		pass
 
 setup(name = 'gnewsclient',
-      version = '1.0.2',
+      version = '1.1.0',
       classifiers = [
         'Development Status :: 4 - Beta',
         'License :: OSI Approved :: MIT License',
@@ -30,7 +30,12 @@ def readme():
       author = 'Nikhil Kumar Singh',
       author_email = 'nikhilksingh97@gmail.com',
       license = 'MIT',
-      packages = ['gnewsclient'],
-      install_requires = ['requests', 'bs4', 'html5lib'], 
+      packages = ['gnewsclient', 'gnewsclient.scripts'],
+      install_requires = ['requests', 'bs4', 'html5lib', 'Click'], 
       include_package_data = True,
-      zip_safe = False)
+      zip_safe = False,
+      entry_points='''
+        [console_scripts]
+        gnews=gnewsclient.scripts.gnews:cli
+      ''',
+      )