Scrape Google Play Search Apps in Python
Hey guys, just in case anyone wants to scrape Google Play Store App Search 👀

Full code: 

```python
from bs4 import BeautifulSoup
from serpapi import GoogleSearch
import requests, json, lxml, re, os


def bs4_scrape_all_google_play_store_search_apps(
                                          query: str, 
                                          filter_by: str = "apps",
                                          country: str = "US"):
    # https://docs.python-requests.org/en/master/user/quickstart/#passing-parameters-in-urls
    params = {
        "q": query,     # search query
        "gl": country,  # country of the search. Different country display different apps.
        "c": filter_by  # filter to display list of apps. Other filters: apps, books, movies
    }

    # https://docs.python-requests.org/en/master/user/quickstart/#custom-headers
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.79 Safari/537.36",
    }

    html = requests.get("https://play.google.com/store/search", params=params, headers=headers, timeout=30)
    soup = BeautifulSoup(html.text, "lxml")

    apps_data = []

    for app in soup.select(".mpg5gc"):
        title = app.select_one(".nnK0zc").text
        company = app.select_one(".b8cIId.KoLSrc").text
        description = app.select_one(".b8cIId.f5NCO a").text
        app_link = f'https://play.google.com{app.select_one(".b8cIId.Q9MA7b a")["href"]}'
        developer_link = f'https://play.google.com{app.select_one(".b8cIId.KoLSrc a")["href"]}'
        app_id = app.select_one(".b8cIId a")["href"].split("id=")[1]
        developer_id = app.select_one(".b8cIId.KoLSrc a")["href"].split("id=")[1]
        
        try:
            # https://regex101.com/r/SZLPRp/1
            rating = re.search(r"\d{1}\.\d{1}", app.select_one(".pf5lIe div[role=img]")["aria-label"]).group()
        except:
            rating = None
        
        thumbnail = app.select_one(".yNWQ8e img")["data-src"]
        
        apps_data.append({
            "title": title,
            "company": company,
            "description": description,
            "rating": float(rating) if rating else rating, # float if rating is not None else rating or None
            "app_link": app_link,
            "developer_link": developer_link,
            "app_id": app_id,
            "developer_id": developer_id,
            "thumbnail": thumbnail
        })        

    print(json.dumps(apps_data, indent=2, ensure_ascii=False))
    
bs4_scrape_all_google_play_store_search_apps(query="maps", filter_by="apps", country="US")




def serpapi_scrape_all_google_play_store_apps():
    params = {
        "api_key": os.getenv("API_KEY"),  # your serpapi api key
        "engine": "google_play",          # search engine
        "hl": "en",                       # language
        "store": "apps",                  # apps search
        "gl": "us",                       # contry to search from. Different country displays different.
        "q": "maps"                       # search qeury
    }

    search = GoogleSearch(params)  # where data extracts
    results = search.get_dict()    # JSON -&gt; Python dictionary

    apps_data = []

    for apps in results["organic_results"]:
        for app in apps["items"]:
            apps_data.append({
                "title": app.get("title"),
                "link": app.get("link"),
                "description": app.get("description"),
                "product_id": app.get("product_id"),
                "rating": app.get("rating"),
                "thumbnail": app.get("thumbnail"),
                })

    print(json.dumps(apps_data, indent=2, ensure_ascii=False))
```

Output from DIY solution:

```json
[
  {
    "title": "Google Maps",
    "company": "Google LLC",
    "description": "Real-time GPS navigation &amp; local suggestions for food, events, &amp; activities",
    "rating": 3.9,
    "app_link": "https://play.google.com/store/apps/details?id=com.google.android.apps.maps",
    "developer_link": "https://play.google.com/store/apps/dev?id=5700313618786177705",
    "app_id": "com.google.android.apps.maps",
    "developer_id": "5700313618786177705",
    "thumbnail": "https://play-lh.googleusercontent.com/Kf8WTct65hFJxBUDm5E-EpYsiDoLQiGGbnuyP6HBNax43YShXti9THPon1YKB6zPYpA=s128-rw"
  },
  {
    "title": "Google Maps Go",
    "company": "Google LLC",
    "description": "Get real-time traffic, directions, search and find places",
    "rating": 4.3,
    "app_link": "https://play.google.com/store/apps/details?id=com.google.android.apps.mapslite",
    "developer_link": "https://play.google.com/store/apps/dev?id=5700313618786177705",
    "app_id": "com.google.android.apps.mapslite",
    "developer_id": "5700313618786177705",
    "thumbnail": "https://play-lh.googleusercontent.com/0uRNRSe4iS6nhvfbBcoScHcBTx1PMmxkCx8rrEsI2UQcQeZ5ByKz8fkhwRqR3vttOg=s128-rw"
  },
  {
    "title": "Waze - GPS, Maps, Traffic Alerts &amp; Live Navigation",
    "company": "Waze",
    "description": "Save time on every drive. Waze tells you about traffic, police, crashes &amp; more",
    "rating": 4.4,
    "app_link": "https://play.google.com/store/apps/details?id=com.waze",
    "developer_link": "https://play.google.com/store/apps/developer?id=Waze",
    "app_id": "com.waze",
    "developer_id": "Waze",
    "thumbnail": "https://play-lh.googleusercontent.com/muSOyE55_Ra26XXx2IiGYqXduq7RchMhosFlWGc7wCS4I1iQXb7BAnnjEYzqcUYa5oo=s128-rw"
  }, ... other results
]
```

Full blog post with step-by-step explanation: https://serpapi.com/blog/scrape-google-play-search-apps-in-python/