--- title: step by step guide tell you how to build a website like apkmirror published: true description: a complete guide to build website like apkmirror tags: freeapkdownload cover_image: https://upload-images.jianshu.io/upload_images/1355291-8ce15afd3ce3e415.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/1240 --- There are many [free apk download](https://idoras.com) websites such as apkmirror, today i will tell you how to build a website like apkmirror, the programming language i used is [node.js](https://nodejs.org/en/), the database i used is [mongodb](https://www.mongodb.com/), search engine used is [elasticsearch](https://elastic.co/), the web framework i use is [eggjs](https://eggjs.org/). **Part I analysis apk file and save data** The tool we need is appt, you can install it with **sudo apt-get install aapt** on ubuntu, the command to analysis apk file is **aapt dump badging test.apk**, as we need execute the command in node.js script so i use execSync to call the command, the code block is below. ``` const { execSync } = require('child_process'); const fs = require("fs"); const md5 = require('md5'); const { sleep, getDir } = require('../util'); const analysis = (output) => { const res = {}; let out = output.toString().replace(/'/g, ''); out = out.split('\n'); let tmp = out[0].split(' '); res.package = tmp[1].split('=')[1]; res.versionCode = tmp[2].split('=')[1]; res.versionName = tmp[3].split('=')[1]; tmp = out[2].split(':') if (tmp[0] == 'sdkVersion') { res.sdkVersion = tmp[1]; } tmp = out[3].split(':'); if (tmp[0] == 'targetSdkVersion') { res.targetSdkVersion = tmp[1]; } tmp = out[out.length - 2].split(':'); if (tmp[0] == 'native-code') { res.variant = tmp[1].trim().replace(/ /g, '|'); tmp = out[out.length - 6].split(':'); res.screen = tmp[1].trim(); tmp = out[out.length - 4].split(':'); res.locales = tmp[1].trim(); } else { res.variant = 'noarch'; tmp = out[out.length - 5].split(':'); res.screen = tmp[1].trim(); tmp = out[out.length - 3].split(':'); res.locales = tmp[1].trim(); } return res; } const filters = ['.', '..']; const run = async () => { while (true) { const dirs = fs.readdirSync(getDir('tmp')); for (const dir of dirs) { if (!filters.includes(dir)) { const name = getDir('tmp', dir); try { const output = execSync(`/usr/bin/aapt dump badging ${name}`); const res = analysis(output); const stats = fs.statSync(name); res.fileSize = stats.size; const newName = getDir('apk', `${res.package}_${res.versionCode}_idoras.com.apk`); const tasks = []; const url = `https://play.google.com/store/apps/details?id=${res.package}`; tasks.push({ _id: md5(url), url, host: 'play.google.com', type: 'detail', package: res.package, done: 0 }); await global.db.collection('tasks').insertMany(tasks, { ordered: false }).catch(() => {}); const one = await global.db.collection('apk').findOne({ _id: res.package }).catch(() => {}); if (one) { const a = +one.versionCode; const b = +res.versionCode; if (b > a) { await global.db.collection('apk').updateOne({ _id: res.package }, { $set:{ udate: Date.now(), versionCode: res.versionCode, versionName: res.versionName, done: 1 }}).catch(() => {}); } } else { await global.db.collection('apk').insertOne({ _id: res.package, versionCode: res.versionCode, versionName: res.versionName, cdate: Date.now(), udate: Date.now(), done: 0, down: 0 }).catch(() => {}); } res._id = `${res.package}_${res.versionCode}`; await global.db.collection('variant').insertOne({ ...res, cdate: Date.now() }).catch(() => {}); fs.renameSync(name, newName); } catch (err) { console.error(err); fs.unlinkSync(name); } } } if (dirs.length == 0) { console.log('no apk files to analysis'); await sleep(60000); } } }; module.exports = run; ``` The **getDir** and **sleep** is util function, you can find the source code below. ``` const req = require('request'); const path = require('path'); const os = require('os'); const agent = new require('socks-proxy-agent')('socks5://localhost:1080'); let root = '/data'; const options = { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36' }, timeout: 30000, encoding: null, }; if (os.platform() == 'win32') { options.agent = agent; root = 'd:/data'; } const request = req.defaults(options); const getDir = (...paths) => { paths.unshift(root); return path.join(...paths); }; const sleep = (time) => new Promise((resolve) => { setTimeout(() => { resolve(); }, time); }); const fetch = (url) => new Promise((resolve) => { console.log(`down ${url} started`); request(url, (err, res, body) => { if (res && res.statusCode === 200) { console.log(`down ${url} 200`); resolve(body); } else { console.error(`down ${url} ${res && res.statusCode} ${err}`); if (res && res.statusCode) { resolve(res.statusCode); } else { // ESOCKETTIMEOUT 超时错误返回600 resolve(600); } } }); }); module.exports = { getDir, sleep, fetch }; ``` The main logic is we get basic information from appt, parse the output, get apk id such as **com.moonvideo.android.resso**, and insert a task to get apk logo and images and brief from [play.google.com](https://play.google.com/store/apps), i wrote a little crawler framework, each website with one parser bind with the host, every url have a type, each type bind with a function, the play.google.com’s parser code is below. ``` const { URL } = require('url'); const md5 = require('md5'); const cheerio = require('cheerio'); const slugify = require('slugify'); const resolve = (from, to) => { const resolvedUrl = new URL(to, new URL(from, 'resolve://')); if (resolvedUrl.protocol === 'resolve:') { // `from` is a relative URL. const { pathname, search, hash } = resolvedUrl; return pathname + search + hash; } return resolvedUrl.toString(); } const fns = { detail: async (page) => { const host = page.host; try { const $ = cheerio.load(page.con.toString(), { decodeEntities: false }); const brief = $('div.DWPxHb > span > div:nth-child(1)').html(); const name = $('h1.AHFaub > span').text(); const dev = $('div.qQKdcc > span:nth-child(1) > a').text(); const tag = $('div.qQKdcc > span:nth-child(2) > a').attr('href').split('/')[4]; const images = []; const tasks = []; $('button.Q4vdJd > img').each((i, ele) => { let url = $(ele).attr('src'); if (!url || !url.startsWith('https')) { url = $(ele).attr('data-src') } tasks.push({ _id: md5(url), url, type: 'download', host: (new URL(url)).hostname, done: 0 }); images.push({ _id: md5(url), url }); }); const logourl = $('div.xSyT2c > img').attr('src'); tasks.push({ _id: md5(logourl), url: logourl, type: 'download', host: (new URL(logourl)).hostname, package: page.package, done: 0 }); const res = await global.db.collection('tasks').insertMany(tasks, { ordered: false }).catch(() => {}); res && console.log(`${host}-detail insert ${res.insertedCount} from ${tasks.length} tasks`); await global.db.collection('apk').updateOne({ _id: page.package }, { $set:{ brief, name, dev, devSlug: slugify(dev), tag, tagSlug: slugify(tag), images, done: 1 } }); return 1; } catch (err) { console.error(`${host}-detail parse ${page.url} ${err}`); return 4; } }, run: (page) => { const fn = fns[page.type]; if (fn) { return fn(page); } console.error(`${page.url} parser not found`); return 0; } }; module.exports = fns; ``` every apk we analysis with aapt will send one request to play.google.com to get it’s name, brief, developer info, tag, images, i will use these data to show on the website. now we have the data of apk file, so where to find the apk files? You can download them from [idoras](https://idoras.com) or [apkmirror](https://www.apkmirror.com/) or [apkpure](https://apkpure.com/), also you can write node.js script to download them from website such as [idoras](https://idoras.com). Next we need index these data with elasticsearch so we can use them later, the code is below. ``` const { sleep } = require('../util'); const { Client } = require('@elastic/elasticsearch'); const getApks = (apks) => { const res = []; apks.forEach((apk) => { if(apk.name) { const tmp = {}; res.push({ index: { _index: 'apk', _type: '_doc', _id: apk._id } }); tmp.id = apk._id; tmp.versionCode = apk.versionCode; tmp.versionName = apk.versionName; tmp.cdate = apk.cdate; tmp.udate = apk.udate; tmp.down = apk.down; tmp.brief = apk.brief; tmp.dev = apk.dev; tmp.devSlug = apk.devSlug; tmp.name = apk.name; tmp.tag = apk.tag; res.push(tmp); } }); return res; }; const run = async () => { while (true) { const esclient = new Client({ node: 'http://username:password@ip:9200', maxRetries: 3, requestTimeout: 60000 }); const apks = await global.db.collection('apk').find({ done: 1 }).limit(100).toArray().catch((err) => console.error(err)); const tmp = getApks(apks); if (tmp && tmp.length) { let res = await esclient.bulk({ refresh: true, body: tmp }).catch((err) => console.error(err)); if (res) { const ulist = apks.map((apk) => ({ updateOne: { filter: { _id: apk._id }, update: { $set: { done: 6 } }, upsert: false } })); res = await global.db.collection('apk').bulkWrite(ulist, { ordered: false, w: 1 }).catch(() => {}); res && console.log(`apk:indexer update ${res.modifiedCount} from ${res.matchedCount}`); } } else { console.log('apk:indexer no apk to index'); await sleep(60000); } await esclient.close(); } }; module.exports = run; ``` **Part II build website with eggjs** The template engine i use is [nunjucks](https://nunjucks.bootcss.com/), also i used some eggjs plugins such [egg-elasticsearch2](https://www.npmjs.com/package/egg-elasticsearch2) The logic is easy, at home page we query data from elasticsearch, and render data into nunjucks template, basiclly it’s a list page, also i will cache the hot download apks into **ctx.app.hots**, refresh it every 30 minutes, and show the hot download apks on the right aside, also the html template support responsive design, it will show perfect on mobile system. ``` 'use strict'; const Controller = require('egg').Controller; class HomeController extends Controller { async home() { const { ctx } = this; const params = ctx.params; const page = parseInt(params.page, 10) || 1; if (page < 1) { ctx.redirect('/list/1/', 301); } const start = (page - 1) * ctx.helper.size; const query = { body: [ { index: 'apk', type: '_doc' }, { query: { bool: { must: [{ match_all: {} }, { exists: { field: 'id' } }] } }, highlight: { fields: { name: {} } }, from: start, size: ctx.helper.size, sort: [{ udate: { order: 'desc' } }, '_score' ] }, ], }; const info = {}; const res = await ctx.app.elasticsearch.msearch(query); ctx.helper.getList(res.responses[0], info); let tpage = Math.floor(info.all / ctx.helper.size) + (info.all % ctx.helper.size ? 1 : 0); tpage < 1 && (tpage = 1); if (page > tpage) { if (tpage > 500) { ctx.redirect('/list/500/', 301); } else { ctx.redirect(`/list/${tpage}/`, 301); } } else if (page > 500) { ctx.redirect('/list/500/', 301); } tpage = tpage > 500 ? 500 : tpage; info.pages = ctx.helper.getPages(page, tpage); info.url = ctx.request.url; if (info.url === '/') { info.title = 'Free Apk download online - idoras.com'; info.keywords = 'app download,apk download, free apk download, idoras, apk downloader, android apk download'; info.description = 'Free apk download for Android with idoras APK downloader. NoAds, Faster apk downloads and apk file update speed. Best of all, it\'s free'; info.home = true; } else { info.title = `Free Apk download - page${page} - idoras.com`; info.keywords = `app download,apk download, free apk download, idoras, apk downloader, android apk download,apk list page${page}`; info.description = info.list.map(item => item.name).join(',').substring(0, 150); } if (!ctx.app.hots) { ctx.app.hots = info.hots = await ctx.service.search.hot(); } else { info.hots = ctx.app.hots; } info.slist = ctx.helper.getKeys(); await ctx.render('home.html', { info }); } async version() { const { ctx } = this; const params = ctx.params; const id = params.id; const apk = await ctx.model.Apk.findById(id); const list = await ctx.model.Variant.find({ package: id }); list.sort((a, b) => b.versionCode - a.versionCode); const info = { apk, list }; info.variant = list[0] || {}; info.title = `${apk.name} Free Apk download - idoras.com`; info.keywords = `${apk.name} download, app download,apk download, free apk download, idoras, apk downloader, android apk download`; info.description = `download ${apk.name} free apks online, it's faster and free online, many version to download`; if (!ctx.app.hots) { ctx.app.hots = info.hots = await ctx.service.search.hot(); } else { info.hots = ctx.app.hots; } info.slist = ctx.helper.getKeys(); await ctx.render('version.html', { info }); } async download() { const { ctx } = this; const params = ctx.params; const id = params.id; const version = params.version; const apk = await ctx.model.Apk.findById(id); const variant = await ctx.model.Variant.findById(`${id}_${version}`); const list = await ctx.model.Variant.find({ package: id }); list.sort((a, b) => b.cdate - a.cdate); apk.down = apk.down + 1; apk.save(); ctx.app.elasticsearch.update({ index: 'apk', type: '_doc', id, body: { script: 'ctx._source.down += 1', upsert: { down: 1 } } }, () => { }); const info = { apk, variant, list }; info.title = `${apk.name} ${apk.versionName} Free Apk download online - idoras.com`; info.keywords = `${apk.name} ${apk.versionName} download online, app download,apk download, free apk download, idoras, apk downloader, android apk download`; info.description = `download ${apk.name} ${apk.versionName} free apks online, it's faster and free online, many version to download`; if (!ctx.app.hots) { ctx.app.hots = info.hots = await ctx.service.search.hot(); } else { info.hots = ctx.app.hots; } info.slist = ctx.helper.getKeys(); await ctx.render('download.html', { info }); } async dev() { const { ctx } = this; const params = ctx.params; const devSlug = params.devSlug; const page = parseInt(params.page, 10) || 1; if (page < 1) { ctx.redirect(`/dev/${devSlug}/1/`, 301); } const start = (page - 1) * ctx.helper.size; const info = { devSlug }; const query = { body: [ { index: 'apk', type: '_doc' }, { query: { term: { devSlug } }, from: start, size: ctx.helper.size, sort: [{ udate: { order: 'desc' } }, '_score' ] }, ], }; const res = await ctx.app.elasticsearch.msearch(query); ctx.helper.getList(res.responses[0], info); let tpage = Math.floor(info.all / ctx.helper.size) + (info.all % ctx.helper.size ? 1 : 0); tpage < 1 && (tpage = 1); if (page > tpage) { if (tpage > 500) { ctx.redirect(`/dev/${devSlug}/500/`, 301); } else { ctx.redirect(`/dev/${devSlug}/${tpage}/`, 301); } } else if (page > 500) { ctx.redirect(`/dev/${devSlug}/500/`, 301); } tpage = tpage > 500 ? 500 : tpage; info.pages = ctx.helper.getPages(page, tpage); info.title = `apk developered by ${devSlug} download - idoras.com`; info.keywords = `${devSlug} apks download online, app download,apk download, free apk download, idoras, apk downloader, android apk download`; info.description = `download ${devSlug} free apks online, it's faster and free online, many version to download`; if (!ctx.app.hots) { ctx.app.hots = info.hots = await ctx.service.search.hot(); } else { info.hots = ctx.app.hots; } info.slist = ctx.helper.getKeys(); await ctx.render('dev.html', { info }); } async tag() { const { ctx } = this; const params = ctx.params; const tag = params.tag; const page = parseInt(params.page, 10) || 1; if (page < 1) { ctx.redirect(`/tag/${tag}/1/`, 301); } const start = (page - 1) * ctx.helper.size; const info = { tag }; const query = { body: [ { index: 'apk', type: '_doc' }, { query: { term: { tag } }, from: start, size: ctx.helper.size, sort: [{ udate: { order: 'desc' } }, '_score' ] }, ], }; const res = await ctx.app.elasticsearch.msearch(query); ctx.helper.getList(res.responses[0], info); let tpage = Math.floor(info.all / ctx.helper.size) + (info.all % ctx.helper.size ? 1 : 0); tpage < 1 && (tpage = 1); if (page > tpage) { if (tpage > 500) { ctx.redirect(`/tag/${tag}/500/`, 301); } else { ctx.redirect(`/tag/${tag}/${tpage}/`, 301); } } else if (page > 500) { ctx.redirect(`/tag/${tag}/500/`, 301); } tpage = tpage > 500 ? 500 : tpage; info.pages = ctx.helper.getPages(page, tpage); info.title = `apk tagged with ${tag} download - idoras.com`; info.keywords = `${tag} apks download online, app download,apk download, free apk download, idoras, apk downloader, android apk download`; info.description = `download ${tag} free apks online, it's faster and free online, many version to download`; if (!ctx.app.hots) { ctx.app.hots = info.hots = await ctx.service.search.hot(); } else { info.hots = ctx.app.hots; } info.slist = ctx.helper.getKeys(); await ctx.render('tag.html', { info }); } async search() { const { ctx } = this; const params = ctx.params; const key = params.key; const page = parseInt(params.page, 10) || 1; if (page < 1) { ctx.redirect(`/search/${key}/1/`, 301); } const start = (page - 1) * ctx.helper.size; const info = { key }; const query = { body: [ { index: 'apk', type: '_doc' }, ], }; if (key) { query.body.push({ query: { multi_match: { query: key, fields: [ 'name', 'id', 'brief' ] } }, from: start, size: ctx.helper.size, sort: [ '_score' ] }); } else { query.body.push({ query: { bool: { must: [{ match_all: {} }, { exists: { field: 'id' } }] } }, from: start, size: ctx.helper.size, sort: [ '_score' ] }); } const res = await ctx.app.elasticsearch.msearch(query); key && res.responses[0].hits.hits.length && ctx.helper.putKey(key); ctx.helper.getList(res.responses[0], info); let tpage = Math.floor(info.all / ctx.helper.size) + (info.all % ctx.helper.size ? 1 : 0); tpage < 1 && (tpage = 1); if (page > tpage) { if (tpage > 500) { ctx.redirect(`/search/${key}/500/`, 301); } else { ctx.redirect(`/search/${key}/${tpage}/`, 301); } } else if (page > 500) { ctx.redirect(`/search/${key}/500/`, 301); } tpage = tpage > 500 ? 500 : tpage; info.pages = ctx.helper.getPages(page, tpage); info.title = `search ${key} apks to download online - idoras.com`; info.keywords = `${key} apks download online, app download,apk download, free apk download, idoras, apk downloader, android apk download`; info.description = `download ${key} free apks online, it's faster and free online, many version to download`; if (!ctx.app.hots) { ctx.app.hots = info.hots = await ctx.service.search.hot(); } else { info.hots = ctx.app.hots; } info.slist = ctx.helper.getKeys(); await ctx.render('search.html', { info }); } async re() { const { ctx } = this; const params = ctx.request.body; const key = params.key; if (key) { ctx.redirect(`/search/${encodeURI(key)}/`, 301); } else { ctx.redirect('/search/', 301); } } } module.exports = HomeController; ``` ``` 'use strict'; // eslint-disable-next-line no-unused-vars module.exports = (options, app) => { return async function mobileMiddleware(ctx, next) { const u = ctx.get('user-agent') || ''; const tmp = { trident: u.indexOf('Trident') > -1, // IE内核 presto: u.indexOf('Presto') > -1, // opera内核 webKit: u.indexOf('AppleWebKit') > -1, // 苹果、谷歌内核 gecko: u.indexOf('Gecko') > -1 && u.indexOf('KHTML') === -1, // 火狐内核 mobile: !!u.match(/AppleWebKit.*Mobile.*/), // 是否为移动终端 ios: !!u.match(/\(i[^;]+;( U;)? CPU.+Mac OS X/), // ios终端 android: u.indexOf('Android') > -1 || u.indexOf('Linux') > -1, // android终端或者uc浏览器 iPhone: u.indexOf('iPhone') > -1, // 是否为iPhone或者QQHD浏览器 iPad: u.indexOf('iPad') > -1, // 是否iPad webApp: u.indexOf('Safari') === -1, // 是否web应该程序，没有头部与底部 weixin: u.indexOf('MicroMessenger') > -1, // 是否微信（2015-01-22新增） qq: u.match(/\sQQ/i) === ' qq', // 是否QQ }; if (tmp.mobile || tmp.android || tmp.ios || tmp.weixin) { ctx.request.mobile = true; } await next(); }; }; ``` ``` {{info.title}}

IDORAS

{% if ctx.request.mobile %}

{% endif %}

Latest Upload

{% for item in info.list %}

by {{item.dev}} {{ctx.helper.fdate(item.udate)}}

{% endfor %}

{% if info.slist.length > 0 %}

Hot Searches

{% for item in info.slist %} {{item.key}} {% endfor %}

{% endif %}

Social Medias

Popular In Last 24 Hours

{% for item in info.hots[0].list %}

{% endfor %}

Popular In Last 7 Days

{% for item in info.hots[1].list %}

{% endfor %}

Popular In Last 30 Days

{% for item in info.hots[2].list %}

{% endfor %}

``` That’s the main code i wrote for the website, leave a comment if you want to know more about my code, at last remember the website address [https://idoras.com](https://idoras.com) i built a week ago, and share it with your firends if you find it useful, thanks.

Latest Upload

{{item.name}} {{item.versionName}}

Hot Searches

Social Medias

Popular In Last 24 Hours

{{item.name}} {{item.versionName}}

Popular In Last 7 Days

{{item.name}} {{item.versionName}}

Popular In Last 30 Days

{{item.name}} {{item.versionName}}

freeapkdownload

step by step guide tell you how to build a website like apkmirror

Join Our Newsletter. No Spam, Only the good stuff.