bd_ing

PR0303: Obtención de datos de una API REST

Práctica: Ingesta de Datos con Star Wars API (SWAPI)

1.- Conexión básica y primer dataFrame

import pandas as pd
import requests
try: 
    request = requests.get("https://swapi.dev/api/vehicles")
    request.raise_for_status()
    df = pd.json_normalize(request.json()["results"])
except requests.exceptions.RequestException as e:
    print(e)
df

	name	model	manufacturer	cost_in_credits	length	max_atmosphering_speed	crew	passengers	cargo_capacity	consumables	vehicle_class	pilots	films	created	edited	url
0	Sand Crawler	Digger Crawler	Corellia Mining Corporation	150000	36.8	30	46	30	50000	2 months	wheeled	[]	[https://swapi.dev/api/films/1/, https://swapi...	2014-12-10T15:36:25.724000Z	2014-12-20T21:30:21.661000Z	https://swapi.dev/api/vehicles/4/
1	T-16 skyhopper	T-16 skyhopper	Incom Corporation	14500	10.4	1200	1	1	50	0	repulsorcraft	[]	[https://swapi.dev/api/films/1/]	2014-12-10T16:01:52.434000Z	2014-12-20T21:30:21.665000Z	https://swapi.dev/api/vehicles/6/
2	X-34 landspeeder	X-34 landspeeder	SoroSuub Corporation	10550	3.4	250	1	1	5	unknown	repulsorcraft	[]	[https://swapi.dev/api/films/1/]	2014-12-10T16:13:52.586000Z	2014-12-20T21:30:21.668000Z	https://swapi.dev/api/vehicles/7/
3	TIE/LN starfighter	Twin Ion Engine/Ln Starfighter	Sienar Fleet Systems	unknown	6.4	1200	1	0	65	2 days	starfighter	[]	[https://swapi.dev/api/films/1/, https://swapi...	2014-12-10T16:33:52.860000Z	2014-12-20T21:30:21.670000Z	https://swapi.dev/api/vehicles/8/
4	Snowspeeder	t-47 airspeeder	Incom corporation	unknown	4.5	650	2	0	10	none	airspeeder	[https://swapi.dev/api/people/1/, https://swap...	[https://swapi.dev/api/films/2/]	2014-12-15T12:22:12Z	2014-12-20T21:30:21.672000Z	https://swapi.dev/api/vehicles/14/
5	TIE bomber	TIE/sa bomber	Sienar Fleet Systems	unknown	7.8	850	1	0	none	2 days	space/planetary bomber	[]	[https://swapi.dev/api/films/2/, https://swapi...	2014-12-15T12:33:15.838000Z	2014-12-20T21:30:21.675000Z	https://swapi.dev/api/vehicles/16/
6	AT-AT	All Terrain Armored Transport	Kuat Drive Yards, Imperial Department of Milit...	unknown	20	60	5	40	1000	unknown	assault walker	[]	[https://swapi.dev/api/films/2/, https://swapi...	2014-12-15T12:38:25.937000Z	2014-12-20T21:30:21.677000Z	https://swapi.dev/api/vehicles/18/
7	AT-ST	All Terrain Scout Transport	Kuat Drive Yards, Imperial Department of Milit...	unknown	2	90	2	0	200	none	walker	[https://swapi.dev/api/people/13/]	[https://swapi.dev/api/films/2/, https://swapi...	2014-12-15T12:46:42.384000Z	2014-12-20T21:30:21.679000Z	https://swapi.dev/api/vehicles/19/
8	Storm IV Twin-Pod cloud car	Storm IV Twin-Pod	Bespin Motors	75000	7	1500	2	0	10	1 day	repulsorcraft	[]	[https://swapi.dev/api/films/2/]	2014-12-15T12:58:50.530000Z	2014-12-20T21:30:21.681000Z	https://swapi.dev/api/vehicles/20/
9	Sail barge	Modified Luxury Sail Barge	Ubrikkian Industries Custom Vehicle Division	285000	30	100	26	500	2000000	Live food tanks	sail barge	[]	[https://swapi.dev/api/films/3/]	2014-12-18T10:44:14.217000Z	2014-12-20T21:30:21.684000Z	https://swapi.dev/api/vehicles/24/

2.- Gestión de paginación

def request_people(df, url):
    try: 
        request = requests.get(url)
        request.raise_for_status()
        json = request.json()
        df = pd.concat([df, pd.json_normalize(json["results"])], ignore_index=True)
        next = json['next']
        if next:
            df = request_people(df, next)
    except requests.exceptions.RequestException as e:
        print(e)
    return df
people_df = request_people(pd.DataFrame({}), "https://swapi.dev/api/people/")
people_df

	name	height	mass	hair_color	skin_color	eye_color	birth_year	gender	homeworld	films	species	vehicles	starships	created	edited	url
0	Luke Skywalker	172	77	blond	fair	blue	19BBY	male	https://swapi.dev/api/planets/1/	[https://swapi.dev/api/films/1/, https://swapi...	[]	[https://swapi.dev/api/vehicles/14/, https://s...	[https://swapi.dev/api/starships/12/, https://...	2014-12-09T13:50:51.644000Z	2014-12-20T21:17:56.891000Z	https://swapi.dev/api/people/1/
1	C-3PO	167	75	n/a	gold	yellow	112BBY	n/a	https://swapi.dev/api/planets/1/	[https://swapi.dev/api/films/1/, https://swapi...	[https://swapi.dev/api/species/2/]	[]	[]	2014-12-10T15:10:51.357000Z	2014-12-20T21:17:50.309000Z	https://swapi.dev/api/people/2/
2	R2-D2	96	32	n/a	white, blue	red	33BBY	n/a	https://swapi.dev/api/planets/8/	[https://swapi.dev/api/films/1/, https://swapi...	[https://swapi.dev/api/species/2/]	[]	[]	2014-12-10T15:11:50.376000Z	2014-12-20T21:17:50.311000Z	https://swapi.dev/api/people/3/
3	Darth Vader	202	136	none	white	yellow	41.9BBY	male	https://swapi.dev/api/planets/1/	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[https://swapi.dev/api/starships/13/]	2014-12-10T15:18:20.704000Z	2014-12-20T21:17:50.313000Z	https://swapi.dev/api/people/4/
4	Leia Organa	150	49	brown	light	brown	19BBY	female	https://swapi.dev/api/planets/2/	[https://swapi.dev/api/films/1/, https://swapi...	[]	[https://swapi.dev/api/vehicles/30/]	[]	2014-12-10T15:20:09.791000Z	2014-12-20T21:17:50.315000Z	https://swapi.dev/api/people/5/
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
77	Grievous	216	159	none	brown, white	green, yellow	unknown	male	https://swapi.dev/api/planets/59/	[https://swapi.dev/api/films/6/]	[https://swapi.dev/api/species/36/]	[https://swapi.dev/api/vehicles/60/]	[https://swapi.dev/api/starships/74/]	2014-12-20T19:43:53.348000Z	2014-12-20T21:17:50.488000Z	https://swapi.dev/api/people/79/
78	Tarfful	234	136	brown	brown	blue	unknown	male	https://swapi.dev/api/planets/14/	[https://swapi.dev/api/films/6/]	[https://swapi.dev/api/species/3/]	[]	[]	2014-12-20T19:46:34.209000Z	2014-12-20T21:17:50.491000Z	https://swapi.dev/api/people/80/
79	Raymus Antilles	188	79	brown	light	brown	unknown	male	https://swapi.dev/api/planets/2/	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[]	2014-12-20T19:49:35.583000Z	2014-12-20T21:17:50.493000Z	https://swapi.dev/api/people/81/
80	Sly Moore	178	48	none	pale	white	unknown	female	https://swapi.dev/api/planets/60/	[https://swapi.dev/api/films/5/, https://swapi...	[]	[]	[]	2014-12-20T20:18:37.619000Z	2014-12-20T21:17:50.496000Z	https://swapi.dev/api/people/82/
81	Tion Medon	206	80	none	grey	black	unknown	male	https://swapi.dev/api/planets/12/	[https://swapi.dev/api/films/6/]	[https://swapi.dev/api/species/37/]	[]	[]	2014-12-20T20:35:04.260000Z	2014-12-20T21:17:50.498000Z	https://swapi.dev/api/people/83/

82 rows × 16 columns

3.- Cruce de datos

reduced_people_df = people_df.head(20)
already_queried_url = dict()
def query_homeworld_info(url):
    if url in already_queried_url.values():
        return already_queried_url[url]
    try:
        response = requests.get(url)
        response.raise_for_status()
        df = pd.json_normalize(response.json())
        info = (df["name"][0], df["terrain"], df["population"][0])
        already_queried_url[url] = info
        return info
    except requests.exceptions.RequestException as es:
        print(e)
    pass
reduced_people_df["homeworld"] = reduced_people_df["homeworld"].apply(query_homeworld_info)
reduced_people_df

/tmp/ipykernel_34906/3644998964.py:16: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  reduced_people_df["homeworld"] = reduced_people_df["homeworld"].apply(query_homeworld_info)

	name	height	mass	hair_color	skin_color	eye_color	birth_year	gender	homeworld	films	species	vehicles	starships	created	edited	url
0	Luke Skywalker	172	77	blond	fair	blue	19BBY	male	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[https://swapi.dev/api/vehicles/14/, https://s...	[https://swapi.dev/api/starships/12/, https://...	2014-12-09T13:50:51.644000Z	2014-12-20T21:17:56.891000Z	https://swapi.dev/api/people/1/
1	C-3PO	167	75	n/a	gold	yellow	112BBY	n/a	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/, https://swapi...	[https://swapi.dev/api/species/2/]	[]	[]	2014-12-10T15:10:51.357000Z	2014-12-20T21:17:50.309000Z	https://swapi.dev/api/people/2/
2	R2-D2	96	32	n/a	white, blue	red	33BBY	n/a	(Naboo, [grassy hills, swamps, forests, mounta...	[https://swapi.dev/api/films/1/, https://swapi...	[https://swapi.dev/api/species/2/]	[]	[]	2014-12-10T15:11:50.376000Z	2014-12-20T21:17:50.311000Z	https://swapi.dev/api/people/3/
3	Darth Vader	202	136	none	white	yellow	41.9BBY	male	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[https://swapi.dev/api/starships/13/]	2014-12-10T15:18:20.704000Z	2014-12-20T21:17:50.313000Z	https://swapi.dev/api/people/4/
4	Leia Organa	150	49	brown	light	brown	19BBY	female	(Alderaan, [grasslands, mountains], 2000000000)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[https://swapi.dev/api/vehicles/30/]	[]	2014-12-10T15:20:09.791000Z	2014-12-20T21:17:50.315000Z	https://swapi.dev/api/people/5/
5	Owen Lars	178	120	brown, grey	light	blue	52BBY	male	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[]	2014-12-10T15:52:14.024000Z	2014-12-20T21:17:50.317000Z	https://swapi.dev/api/people/6/
6	Beru Whitesun lars	165	75	brown	light	blue	47BBY	female	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[]	2014-12-10T15:53:41.121000Z	2014-12-20T21:17:50.319000Z	https://swapi.dev/api/people/7/
7	R5-D4	97	32	n/a	white, red	red	unknown	n/a	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/]	[https://swapi.dev/api/species/2/]	[]	[]	2014-12-10T15:57:50.959000Z	2014-12-20T21:17:50.321000Z	https://swapi.dev/api/people/8/
8	Biggs Darklighter	183	84	black	light	brown	24BBY	male	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/1/]	[]	[]	[https://swapi.dev/api/starships/12/]	2014-12-10T15:59:50.509000Z	2014-12-20T21:17:50.323000Z	https://swapi.dev/api/people/9/
9	Obi-Wan Kenobi	182	77	auburn, white	fair	blue-gray	57BBY	male	(Stewjon, [grass], unknown)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[https://swapi.dev/api/vehicles/38/]	[https://swapi.dev/api/starships/48/, https://...	2014-12-10T16:16:29.192000Z	2014-12-20T21:17:50.325000Z	https://swapi.dev/api/people/10/
10	Anakin Skywalker	188	84	blond	fair	blue	41.9BBY	male	(Tatooine, [desert], 200000)	[https://swapi.dev/api/films/4/, https://swapi...	[]	[https://swapi.dev/api/vehicles/44/, https://s...	[https://swapi.dev/api/starships/39/, https://...	2014-12-10T16:20:44.310000Z	2014-12-20T21:17:50.327000Z	https://swapi.dev/api/people/11/
11	Wilhuff Tarkin	180	unknown	auburn, grey	fair	blue	64BBY	male	(Eriadu, [cityscape], 22000000000)	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[]	2014-12-10T16:26:56.138000Z	2014-12-20T21:17:50.330000Z	https://swapi.dev/api/people/12/
12	Chewbacca	228	112	brown	unknown	blue	200BBY	male	(Kashyyyk, [jungle, forests, lakes, rivers], 4...	[https://swapi.dev/api/films/1/, https://swapi...	[https://swapi.dev/api/species/3/]	[https://swapi.dev/api/vehicles/19/]	[https://swapi.dev/api/starships/10/, https://...	2014-12-10T16:42:45.066000Z	2014-12-20T21:17:50.332000Z	https://swapi.dev/api/people/13/
13	Han Solo	180	80	brown	fair	brown	29BBY	male	(Corellia, [plains, urban, hills, forests], 30...	[https://swapi.dev/api/films/1/, https://swapi...	[]	[]	[https://swapi.dev/api/starships/10/, https://...	2014-12-10T16:49:14.582000Z	2014-12-20T21:17:50.334000Z	https://swapi.dev/api/people/14/
14	Greedo	173	74	n/a	green	black	44BBY	male	(Rodia, [jungles, oceans, urban, swamps], 1300...	[https://swapi.dev/api/films/1/]	[https://swapi.dev/api/species/4/]	[]	[]	2014-12-10T17:03:30.334000Z	2014-12-20T21:17:50.336000Z	https://swapi.dev/api/people/15/
15	Jabba Desilijic Tiure	175	1,358	n/a	green-tan, brown	orange	600BBY	hermaphrodite	(Nal Hutta, [urban, oceans, swamps, bogs], 700...	[https://swapi.dev/api/films/1/, https://swapi...	[https://swapi.dev/api/species/5/]	[]	[]	2014-12-10T17:11:31.638000Z	2014-12-20T21:17:50.338000Z	https://swapi.dev/api/people/16/
16	Wedge Antilles	170	77	brown	fair	hazel	21BBY	male	(Corellia, [plains, urban, hills, forests], 30...	[https://swapi.dev/api/films/1/, https://swapi...	[]	[https://swapi.dev/api/vehicles/14/]	[https://swapi.dev/api/starships/12/]	2014-12-12T11:08:06.469000Z	2014-12-20T21:17:50.341000Z	https://swapi.dev/api/people/18/
17	Jek Tono Porkins	180	110	brown	fair	blue	unknown	male	(Bestine IV, [rocky islands, oceans], 62000000)	[https://swapi.dev/api/films/1/]	[]	[]	[https://swapi.dev/api/starships/12/]	2014-12-12T11:16:56.569000Z	2014-12-20T21:17:50.343000Z	https://swapi.dev/api/people/19/
18	Yoda	66	17	white	green	brown	896BBY	male	(unknown, [unknown], unknown)	[https://swapi.dev/api/films/2/, https://swapi...	[https://swapi.dev/api/species/6/]	[]	[]	2014-12-15T12:26:01.042000Z	2014-12-20T21:17:50.345000Z	https://swapi.dev/api/people/20/
19	Palpatine	170	75	grey	pale	yellow	82BBY	male	(Naboo, [grassy hills, swamps, forests, mounta...	[https://swapi.dev/api/films/2/, https://swapi...	[]	[]	[]	2014-12-15T12:48:05.971000Z	2014-12-20T21:17:50.347000Z	https://swapi.dev/api/people/21/

4.- Expansión de filas (opcional)

df_exploded = reduced_people_df.explode('films')
movies_url = dict()
def check_film_name(url):
    if url in movies_url.values():
        return movies_url[url]
    pass
    try:
        response = requests.get(url)
        response.raise_for_status()
        json = response.json()
        movies_url[url] = json["title"]
        return json["title"]
    except requests.exceptions.RequestException as es:
        print(e)
    pass
df_exploded['films'] = df_exploded['films'].apply(check_film_name)

df_exploded

	name	height	mass	hair_color	skin_color	eye_color	birth_year	gender	homeworld	films	species	vehicles	starships	created	edited	url
0	Luke Skywalker	172	77	blond	fair	blue	19BBY	male	(Tatooine, [desert], 200000)	A New Hope	[]	[https://swapi.dev/api/vehicles/14/, https://s...	[https://swapi.dev/api/starships/12/, https://...	2014-12-09T13:50:51.644000Z	2014-12-20T21:17:56.891000Z	https://swapi.dev/api/people/1/
0	Luke Skywalker	172	77	blond	fair	blue	19BBY	male	(Tatooine, [desert], 200000)	The Empire Strikes Back	[]	[https://swapi.dev/api/vehicles/14/, https://s...	[https://swapi.dev/api/starships/12/, https://...	2014-12-09T13:50:51.644000Z	2014-12-20T21:17:56.891000Z	https://swapi.dev/api/people/1/
0	Luke Skywalker	172	77	blond	fair	blue	19BBY	male	(Tatooine, [desert], 200000)	Return of the Jedi	[]	[https://swapi.dev/api/vehicles/14/, https://s...	[https://swapi.dev/api/starships/12/, https://...	2014-12-09T13:50:51.644000Z	2014-12-20T21:17:56.891000Z	https://swapi.dev/api/people/1/
0	Luke Skywalker	172	77	blond	fair	blue	19BBY	male	(Tatooine, [desert], 200000)	Revenge of the Sith	[]	[https://swapi.dev/api/vehicles/14/, https://s...	[https://swapi.dev/api/starships/12/, https://...	2014-12-09T13:50:51.644000Z	2014-12-20T21:17:56.891000Z	https://swapi.dev/api/people/1/
1	C-3PO	167	75	n/a	gold	yellow	112BBY	n/a	(Tatooine, [desert], 200000)	A New Hope	[https://swapi.dev/api/species/2/]	[]	[]	2014-12-10T15:10:51.357000Z	2014-12-20T21:17:50.309000Z	https://swapi.dev/api/people/2/
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
19	Palpatine	170	75	grey	pale	yellow	82BBY	male	(Naboo, [grassy hills, swamps, forests, mounta...	The Empire Strikes Back	[]	[]	[]	2014-12-15T12:48:05.971000Z	2014-12-20T21:17:50.347000Z	https://swapi.dev/api/people/21/
19	Palpatine	170	75	grey	pale	yellow	82BBY	male	(Naboo, [grassy hills, swamps, forests, mounta...	Return of the Jedi	[]	[]	[]	2014-12-15T12:48:05.971000Z	2014-12-20T21:17:50.347000Z	https://swapi.dev/api/people/21/
19	Palpatine	170	75	grey	pale	yellow	82BBY	male	(Naboo, [grassy hills, swamps, forests, mounta...	The Phantom Menace	[]	[]	[]	2014-12-15T12:48:05.971000Z	2014-12-20T21:17:50.347000Z	https://swapi.dev/api/people/21/
19	Palpatine	170	75	grey	pale	yellow	82BBY	male	(Naboo, [grassy hills, swamps, forests, mounta...	Attack of the Clones	[]	[]	[]	2014-12-15T12:48:05.971000Z	2014-12-20T21:17:50.347000Z	https://swapi.dev/api/people/21/
19	Palpatine	170	75	grey	pale	yellow	82BBY	male	(Naboo, [grassy hills, swamps, forests, mounta...	Revenge of the Sith	[]	[]	[]	2014-12-15T12:48:05.971000Z	2014-12-20T21:17:50.347000Z	https://swapi.dev/api/people/21/

68 rows × 16 columns