
import pandas as pd
import numpy as np
import os
import requests
import json
import bs4

from IPython.display import display, Image


html_string = '''
<html>
    <body>
      <div id="content">
        <h1>Heading here</h1>
        <p>My First paragraph</p>
        <p>My <em>second</em> paragraph</p>
        <hr>
      </div>
      <div id="nav">
        <ul>
          <li>item 1</li>
          <li>item 2</li>
          <li>item 3</li>
        </ul>
      </div>
    </body>
</html>
'''.strip()


soup = bs4.BeautifulSoup(html_string)


type(soup)

bs4.BeautifulSoup


soup.find_all('div')

[<div id="content">
 <h1>Heading here</h1>
 <p>My First paragraph</p>
 <p>My <em>second</em> paragraph</p>
 <hr/>
 </div>,
 <div id="nav">
 <ul>
 <li>item 1</li>
 <li>item 2</li>
 <li>item 3</li>
 </ul>
 </div>]


soup.find_all('li')

[<li>item 1</li>, <li>item 2</li>, <li>item 3</li>]


[x.text for x in soup.find_all('li')]

['item 1', 'item 2', 'item 3']


soup.find('p')

<p>My First paragraph</p>


soup.find('p').text

'My First paragraph'


soup.find('div')

<div id="content">
<h1>Heading here</h1>
<p>My First paragraph</p>
<p>My <em>second</em> paragraph</p>
<hr/>
</div>


soup.find('div').attrs

{'id': 'content'}


soup.find('div').get('id')

'content'


soup

<html>
<body>
<div id="content">
<h1>Heading here</h1>
<p>My First paragraph</p>
<p>My <em>second</em> paragraph</p>
<hr/>
</div>
<div id="nav">
<ul>
<li>item 1</li>
<li>item 2</li>
<li>item 3</li>
</ul>
</div>
</body>
</html>


# While there are multiple 'id' attributes, none of them are in the <html> tag at the top.
soup.get('id')


soup.find('div').get('id')

'content'


fac_response = requests.get('https://datascience.ucsd.edu/about/faculty/faculty/')
fac_response

<Response [200]>


soup = bs4.BeautifulSoup(fac_response.text)


divs = soup.find_all('div', attrs={'data-entry-type': 'individual'})


divs[0]

<div class="cn-list-row cn-list-item vcard individual computer-science-engg data-infrastructure-and-systems faculty faculty-council leadership staff teaching-faculty" data-entry-id="164" data-entry-slug="rajesh-gupta" data-entry-type="individual" id="rajesh-gupta"><div class="cn-entry cn-accordion" id="entry-id-16463ef2326e65f9"><div class="cn-left" style="min-width: 215px;"> <span class="cn-image-style"><span style="display: block; max-width: 100%; width: 215px"><img alt="Photo of Rajesh Gupta" class="cn-image photo" decoding="async" height="215" lazyload="1" loading="lazy" sizes="100vw" srcset="https://sp-ao.shortpixel.ai/client/to_webp,q_lossless,ret_img/https://datascience.ucsd.edu/wp-content/uploads/connections-images/rajesh-gupta/Rajesh-Gupta-Web-dcd52346218888e8014f4cce4c4baa14.jpg 1x" title="Photo of Rajesh Gupta" width="215"/></span></span></div><div class="cn-right"><h3 style="border-bottom: #182A48 1px solid; color:#182A48;"><a href="https://datascience.ucsd.edu/about/faculty/faculty/name/rajesh-gupta/" title="Rajesh Gupta"><span class="fn n notranslate"><span class="given-name">Rajesh</span> <span class="family-name">Gupta</span></span></a></h3><h4 class="title">Distinguished Professor, HDSI Founding Director</h4><div class="cn-excerpt" id="cn-excerpt-16463ef2326e65f9"><p class="detailtext">Professor Gupta’s research interests span topics in embedded and cyber-physical systems with a focus on energy efficiency from algorithms, devices to systems that scale from IC chips, and data centers to built environments such as commercial buildings.</p><p class="detailtext">Gupta received a Bachelor of Technology in electrical engineering from IIT Kanpur, India; a Master of Science in EECS from University of California, Berkeley; and a PhD in electrical engineering from Stanford University, US. Gupta is a Fellow of the IEEE, the ACM and the American Association for the Advancement of Science.</p><span class="cn-link-more"><a href="https://datascience.ucsd.edu/about/faculty/faculty/name/rajesh-gupta/">Learn More</a></span></div><div class="cn-detail cn-hide" id="cn-detail-16463ef2326e65f9"><div class="cn-bio" id="cn-bio-16463ef2326e65f9"><div class="cn-biography"><p class="detailtext">Professor Gupta’s research interests span topics in embedded and cyber-physical systems with a focus on energy efficiency from algorithms, devices to systems that scale from IC chips, and data centers to built environments such as commercial buildings.</p><p class="detailtext">Gupta received a Bachelor of Technology in electrical engineering from IIT Kanpur, India; a Master of Science in EECS from University of California, Berkeley; and a PhD in electrical engineering from Stanford University, US. Gupta is a Fellow of the IEEE, the ACM and the American Association for the Advancement of Science.</p></div></div></div></div></div></div>


divs[0].find('a').get('title')

'Rajesh Gupta'


divs[0].find('h4').text

'Distinguished Professor, HDSI Founding Director'


divs[0].find('div', attrs={'class': 'cn-bio'}).text.strip()

'Professor Gupta’s research interests span topics in embedded and cyber-physical systems with a focus on energy efficiency from algorithms, devices to systems that scale from IC chips, and data centers to built environments such as commercial buildings.Gupta received a Bachelor of Technology in electrical engineering from IIT Kanpur, India; a Master of Science in EECS from University of California, Berkeley; and a PhD in electrical engineering from Stanford University, US. Gupta is a Fellow of the IEEE, the ACM and the American Association for the Advancement of Science.'


names = [div.find('a').get('title') for div in divs]
names[:5]

['Rajesh Gupta',
 'Virginia de Sa',
 'Tiffany Amariuta',
 'Mikio Aoi',
 'Ery Arias-Castro']


titles = []
for div in divs:
    h4 = div.find('h4')
    if h4:
        titles.append(h4.text)
    else:
        titles.append('')


bios = []
for div in divs:
    bio_div = div.find('div', attrs={'class': 'cn-bio'})
    if bio_div:
        bios.append(bio_div.text.strip())
    else:
        bios.append('')


faculty = pd.DataFrame().assign(name=names, title=titles, bio=bios)
faculty.head()


faculty[faculty['title'] == 'Lecturer']


def show_picture(name):
    idx = names.index(name)
    url = divs[idx].find('img').get('srcset')
    url = url.replace('https://sp-ao.shortpixel.ai/client/to_webp,q_lossless,ret_img/', '') \
             .replace(' 1x', '')
    display(Image(url))


show_picture('Suraj Rampure')


def download_page(i):
    url = f'https://quotes.toscrape.com/page/{i}'
    request = requests.get(url)
    return bs4.BeautifulSoup(request.text)


soup = download_page(5)


divs = soup.find_all('div', attrs={'class': 'quote'})


divs[0]

<div class="quote" itemscope="" itemtype="http://schema.org/CreativeWork">
<span class="text" itemprop="text">“A reader lives a thousand lives before he dies, said Jojen. The man who never reads lives only one.”</span>
<span>by <small class="author" itemprop="author">George R.R. Martin</small>
<a href="/author/George-R-R-Martin">(about)</a>
</span>
<div class="tags">
            Tags:
            <meta class="keywords" content="read,readers,reading,reading-books" itemprop="keywords"/>
<a class="tag" href="/tag/read/page/1/">read</a>
<a class="tag" href="/tag/readers/page/1/">readers</a>
<a class="tag" href="/tag/reading/page/1/">reading</a>
<a class="tag" href="/tag/reading-books/page/1/">reading-books</a>
</div>
</div>


divs[0].find('span', attrs={'class': 'text'}).text

'“A reader lives a thousand lives before he dies, said Jojen. The man who never reads lives only one.”'


divs[0].find('small', attrs={'class': 'author'}).text

'George R.R. Martin'


divs[0].find('a').get('href')

'/author/George-R-R-Martin'


divs[0].find('meta', attrs={'class': 'keywords'}).get('content')

'read,readers,reading,reading-books'


def process_quote(div):
    quote = div.find('span', attrs={'class': 'text'}).text
    author = div.find('small', attrs={'class': 'author'}).text
    author_url = 'https://quotes.toscrape.com' + div.find('a').get('href')
    tags = div.find('meta', attrs={'class': 'keywords'}).get('content')
    
    return pd.Series({'quote': quote, 'author': author, 'author_url': author_url, 'tags': tags})


process_quote(divs[3])

quote         “If you can make a woman laugh, you can make h...
author                                           Marilyn Monroe
author_url    https://quotes.toscrape.com/author/Marilyn-Monroe
tags                                                 girls,love
dtype: object


def process_page(divs):
    return pd.DataFrame([process_quote(div) for div in divs])


process_page(divs)


def quote_df(n):
    '''Returns a DataFrame containing the quotes on the first n pages of https://quotes.toscrape.com/.'''
    dfs = []
    for i in range(1, n + 1):
        # Download page n and create a BeautifulSoup object.
        soup = download_page(i)
        
        # Create DataFrame using the information in that page.
        divs = soup.find_all('div', attrs={'class': 'quote'})
        df = process_page(divs)
        
        # Append DataFrame to dfs.
        dfs.append(df)
        
    # Stitch all DataFrames together.
    return pd.concat(dfs).reset_index(drop=True)


first_three_pages = quote_df(3)
first_three_pages.head()


f = open(os.path.join('data', 'quotes2scrape.json'))


json.loads(f.readline())

{'auth_url': 'http://quotes.toscrape.com/author/Albert-Einstein',
 'quote_auth': 'Albert Einstein',
 'quote_text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
 'bio': 'In 1879, Albert Einstein was born in Ulm, Germany. He completed his Ph.D. at the University of Zurich by 1909. His 1905 paper explaining the photoelectric effect, the basis of electronics, earned him the Nobel Prize in 1921. His first paper on Special Relativity Theory, also published in 1905, changed the world. After the rise of the Nazi party, Einstein made Princeton his permanent home, becoming a U.S. citizen in 1940. Einstein, a pacifist during World War I, stayed a firm proponent of social justice and responsibility. He chaired the Emergency Committee of Atomic Scientists, which organized to alert the public to the dangers of atomic warfare.At a symposium, he advised: "In their struggle for the ethical good, teachers of religion must have the stature to give up the doctrine of a personal God, that is, give up that source of fear and hope which in the past placed such vast power in the hands of priests. In their labors they will have to avail themselves of those forces which are capable of cultivating the Good, the True, and the Beautiful in humanity itself. This is, to be sure a more difficult but an incomparably more worthy task . . . " ("Science, Philosophy and Religion, A Symposium," published by the Conference on Science, Philosophy and Religion in their Relation to the Democratic Way of Life, Inc., New York, 1941). In a letter to philosopher Eric Gutkind, dated Jan. 3, 1954, Einstein stated: "The word god is for me nothing more than the expression and product of human weaknesses, the Bible a collection of honorable, but still primitive legends which are nevertheless pretty childish. No interpretation no matter how subtle can (for me) change this," (The Guardian, "Childish superstition: Einstein\'s letter makes view of religion relatively clear," by James Randerson, May 13, 2008). D. 1955.While best known for his mass–energy equivalence formula E = mc2 (which has been dubbed "the world\'s most famous equation"), he received the 1921 Nobel Prize in Physics "for his services to theoretical physics, and especially for his discovery of the law of the photoelectric effect". The latter was pivotal in establishing quantum theory.Einstein thought that Newtonion mechanics was no longer enough to reconcile the laws of classical mechanics with the laws of the electromagnetic field. This led to the development of his special theory of relativity. He realized, however, that the principle of relativity could also be extended to gravitational fields, and with his subsequent theory of gravitation in 1916, he published a paper on the general theory of relativity. He continued to deal with problems of statistical mechanics and quantum theory, which led to his explanations of particle theory and the motion of molecules. He also investigated the thermal properties of light which laid the foundation of the photon theory of light.He was visiting the United States when Adolf Hitler came to power in 1933 and did not go back to Germany. On the eve of World War II, he endorsed a letter to President Franklin D. Roosevelt alerting him to the potential development of "extremely powerful bombs of a new type" and recommending that the U.S. begin similar research. This eventually led to what would become the Manhattan Project. Einstein supported defending the Allied forces, but largely denounced the idea of using the newly discovered nuclear fission as a weapon. Later, with Bertrand Russell, Einstein signed the Russell–Einstein Manifesto, which highlighted the danger of nuclear weapons. Einstein was affiliated with the Institute for Advanced Study in Princeton, New Jersey, until his death in 1955.His great intellectual achievements and originality have made the word "Einstein" synonymous with genius.More: http://en.wikipedia.org/wiki/Albert_E...http://www.nobelprize.org/nobel_prize...',
 'dob': 'March 14, 1879',
 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}


L = [json.loads(x) for x in open(os.path.join('data', 'quotes2scrape.json'))]


df = pd.DataFrame(L)
df.head()


df['tags'].iloc[0]

['change', 'deep-thoughts', 'thinking', 'world']


df.to_csv('out.csv')


df_again = pd.read_csv('out.csv')
df_again.head()


df_again['tags'].iloc[0]

"['change', 'deep-thoughts', 'thinking', 'world']"


def flatten_tags(taglist):
    return pd.Series({k:1 for k in taglist}, dtype=float)

tags = df['tags'].apply(flatten_tags).fillna(0).astype(int)
tags.head()


df_full = pd.concat([df, tags], axis=1).drop(columns='tags')
df_full.head()


df_full[df_full['inspirational'] == 1].head()

	quote	author	author_url	tags
0	“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”	Albert Einstein	https://quotes.toscrape.com/author/Albert-Einstein	change,deep-thoughts,thinking,world
1	“It is our choices, Harry, that show what we truly are, far more than our abilities.”	J.K. Rowling	https://quotes.toscrape.com/author/J-K-Rowling	abilities,choices
2	“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”	Albert Einstein	https://quotes.toscrape.com/author/Albert-Einstein	inspirational,life,live,miracle,miracles

	quote	author	author_url	tags
0	“A reader lives a thousand lives before he die...	George R.R. Martin	https://quotes.toscrape.com/author/George-R-R-...	read,readers,reading,reading-books
1	“You can never get a cup of tea large enough o...	C.S. Lewis	https://quotes.toscrape.com/author/C-S-Lewis	books,inspirational,reading,tea
2	“You believe lies so you eventually learn to t...	Marilyn Monroe	https://quotes.toscrape.com/author/Marilyn-Monroe
3	“If you can make a woman laugh, you can make h...	Marilyn Monroe	https://quotes.toscrape.com/author/Marilyn-Monroe	girls,love
4	“Life is like riding a bicycle. To keep your b...	Albert Einstein	https://quotes.toscrape.com/author/Albert-Eins...	life,simile
5	“The real lover is the man who can thrill you ...	Marilyn Monroe	https://quotes.toscrape.com/author/Marilyn-Monroe	love
6	“A wise girl kisses but doesn't love, listens ...	Marilyn Monroe	https://quotes.toscrape.com/author/Marilyn-Monroe	attributed-no-source
7	“Only in the darkness can you see the stars.”	Martin Luther King Jr.	https://quotes.toscrape.com/author/Martin-Luth...	hope,inspirational
8	“It matters not what someone is born, but what...	J.K. Rowling	https://quotes.toscrape.com/author/J-K-Rowling	dumbledore
9	“Love does not begin and end the way we seem t...	James Baldwin	https://quotes.toscrape.com/author/James-Baldwin	love

	quote	author	author_url	tags
0	“The world as we have created it is a process ...	Albert Einstein	https://quotes.toscrape.com/author/Albert-Eins...	change,deep-thoughts,thinking,world
1	“It is our choices, Harry, that show what we t...	J.K. Rowling	https://quotes.toscrape.com/author/J-K-Rowling	abilities,choices
2	“There are only two ways to live your life. On...	Albert Einstein	https://quotes.toscrape.com/author/Albert-Eins...	inspirational,life,live,miracle,miracles
3	“The person, be it gentleman or lady, who has ...	Jane Austen	https://quotes.toscrape.com/author/Jane-Austen	aliteracy,books,classic,humor
4	“Imperfection is beauty, madness is genius and...	Marilyn Monroe	https://quotes.toscrape.com/author/Marilyn-Monroe	be-yourself,inspirational

	auth_url	quote_auth	quote_text	bio	dob	tags
0	http://quotes.toscrape.com/author/Albert-Einstein	Albert Einstein	“The world as we have created it is a process ...	In 1879, Albert Einstein was born in Ulm, Germ...	March 14, 1879	[change, deep-thoughts, thinking, world]
1	http://quotes.toscrape.com/author/J-K-Rowling	J.K. Rowling	“It is our choices, Harry, that show what we t...	See also: Robert GalbraithAlthough she writes ...	July 31, 1965	[abilities, choices]
2	http://quotes.toscrape.com/author/Albert-Einstein	Albert Einstein	“There are only two ways to live your life. On...	In 1879, Albert Einstein was born in Ulm, Germ...	March 14, 1879	[inspirational, life, live, miracle, miracles]
3	http://quotes.toscrape.com/author/Jane-Austen	Jane Austen	“The person, be it gentleman or lady, who has ...	Jane Austen was an English novelist whose work...	December 16, 1775	[aliteracy, books, classic, humor]
4	http://quotes.toscrape.com/author/Marilyn-Monroe	Marilyn Monroe	“Imperfection is beauty, madness is genius and...	Marilyn Monroe (born Norma Jeane Mortenson; Ju...	June 01, 1926	[be-yourself, inspirational]

	Unnamed: 0	auth_url	quote_auth	quote_text	bio	dob	tags
0	0	http://quotes.toscrape.com/author/Albert-Einstein	Albert Einstein	“The world as we have created it is a process ...	In 1879, Albert Einstein was born in Ulm, Germ...	March 14, 1879	['change', 'deep-thoughts', 'thinking', 'world']
1	1	http://quotes.toscrape.com/author/J-K-Rowling	J.K. Rowling	“It is our choices, Harry, that show what we t...	See also: Robert GalbraithAlthough she writes ...	July 31, 1965	['abilities', 'choices']
2	2	http://quotes.toscrape.com/author/Albert-Einstein	Albert Einstein	“There are only two ways to live your life. On...	In 1879, Albert Einstein was born in Ulm, Germ...	March 14, 1879	['inspirational', 'life', 'live', 'miracle', '...
3	3	http://quotes.toscrape.com/author/Jane-Austen	Jane Austen	“The person, be it gentleman or lady, who has ...	Jane Austen was an English novelist whose work...	December 16, 1775	['aliteracy', 'books', 'classic', 'humor']
4	4	http://quotes.toscrape.com/author/Marilyn-Monroe	Marilyn Monroe	“Imperfection is beauty, madness is genius and...	Marilyn Monroe (born Norma Jeane Mortenson; Ju...	June 01, 1926	['be-yourself', 'inspirational']

Lecture 16 – More Parsing Examples¶

DSC 80, Winter 2023¶

📣 Announcements¶

Agenda¶

Parsing HTML using Beautiful Soup¶

`BeautifulSoup` objects¶

Finding elements in a tree¶

Using `find_all`¶

Node attributes¶

Example: Scraping the HDSI Faculty page¶

Example¶

Example: Scraping quotes¶

Example: Scraping quotes¶

The plan¶

Downloading a single page¶

Parsing a single page¶

Putting it all together¶

Key takeaways¶

Nested vs. flat data formats¶

Nested vs. flat data formats¶

Aside: JSON Crack¶

Example: Scraping quotes, again¶

One-hot encoding¶

Summary, next time¶

Summary¶

Next time¶

	name	title	bio
0	Rajesh Gupta	Distinguished Professor, HDSI Founding Director	Professor Gupta’s research interests span topi...
1	Virginia de Sa	Professor, HDSI Associate Director	Associate Director de Sa is a leader in the fi...
2	Tiffany Amariuta	Assistant Professor	PhD: Harvard UniversityBefore starting her lab...
3	Mikio Aoi	Assistant Professor	Dr. Aoi is a computational neuroscientist inte...
4	Ery Arias-Castro	Professor	Ery Arias-Castro received his Ph.D. in Statist...

	name	title	bio
21	Marina Langlois	Lecturer
29	Suraj Rampure	Lecturer	Suraj Rampure is a Lecturer in the Halıcıoğlu ...
40	Janine Tiefenbruck	Lecturer	Tiefenbruck focuses her mathematics background...
46	Frank Wuerthwein	Lecturer	Frank W. Wuerthwein is the Director of the San...

	change	deep-thoughts	thinking	world	abilities	choices	inspirational	life	live	miracle	...
0	1	1	1	1	0	0	0	0	0	0	...
1	0	0	0	0	1	1	0	0	0	0	...
2	0	0	0	0	0	0	1	1	1	1	...
3	0	0	0	0	0	0	0	0	0	0	...
4	0	0	0	0	0	0	1	0	0	0	...

	change	deep-thoughts	thinking	world	abilities	choices	inspirational	life	live	miracle	...
0	1	1	1	1	0	0	0	0	0	0	...
1	0	0	0	0	1	1	0	0	0	0	...
2	0	0	0	0	0	0	1	1	1	1	...
3	0	0	0	0	0	0	0	0	0	0	...
4	0	0	0	0	0	0	1	0	0	0	...

Lecture 16 – More Parsing Examples¶

DSC 80, Winter 2023¶

📣 Announcements¶

Agenda¶

Parsing HTML using Beautiful Soup¶

BeautifulSoup objects¶

Finding elements in a tree¶

Using find_all¶

Node attributes¶

Example: Scraping the HDSI Faculty page¶

Example¶

Example: Scraping quotes¶

Example: Scraping quotes¶

The plan¶

Downloading a single page¶

Parsing a single page¶

Putting it all together¶

Key takeaways¶

Nested vs. flat data formats¶

Nested vs. flat data formats¶

Aside: JSON Crack¶

Example: Scraping quotes, again¶

One-hot encoding¶

Summary, next time¶

Summary¶

Next time¶

`BeautifulSoup` objects¶

Using `find_all`¶

	change	deep-thoughts	thinking	world	abilities	choices	inspirational	life	live	miracle	...
0	1	1	1	1	0	0	0	0	0	0	...
1	0	0	0	0	1	1	0	0	0	0	...
2	0	0	0	0	0	0	1	1	1	1	...
3	0	0	0	0	0	0	0	0	0	0	...
4	0	0	0	0	0	0	1	0	0	0	...