Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package com.vertigo.crawler
- import java.util.Random
- import com.vertigo.model.enums.{EntityType, Provider}
- import com.vertigo.model.json.sync.CrawlerTask
- import com.vertigo.test.util.RandomUtil
- import com.wrapper.spotify.models._
- import rx.lang.scala.Observable
- import scala.collection.JavaConverters._
- import scala.collection.{mutable ⇒ m}
- import scala.util
- /*
- Add to pom
- <plugin>
- <groupId>org.scala-tools</groupId>
- <artifactId>maven-scala-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <jvmArgs>
- <jvmArg>-Xms64m</jvmArg>
- <jvmArg>-Xmx1024m</jvmArg>
- </jvmArgs>
- </configuration>
- </plugin>
- <dependency>
- <groupId>io.reactivex</groupId>
- <artifactId>rxjava</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.11.8</version>
- </dependency>
- <dependency>
- <groupId>io.reactivex</groupId>
- <artifactId>rxscala_2.11</artifactId>
- <version>0.26.3</version>
- </dependency>
- */
- object Example extends App {
- val random = new Random(69)
- val ru = new RandomUtil(random)
- val genres = m.Buffer(
- "aggrotech", "ebm", "electro-industrial", "futurepop",
- "ambient psychill", "psychill", "ambient", "metal",
- "crossover thrash", "death metal", "groove metal",
- "nwobhm", "speed metal", "thrash metal", "digital hardcore",
- "industrial", "industrial metal", "industrial rock",
- "neue deutsche harte", "album rock", "alternative metal",
- "classic rock", "hard rock", "nu metal", "power metal",
- "rock", "wrestling")
- val markets = m.Buffer(
- "AD", "AR", "AT", "AU", "BE", "BG", "BO", "BR", "CA", "CH",
- "CL", "CO", "CR", "CY", "CZ", "DE", "DK", "DO", "EC", "EE",
- "ES", "FI", "FR", "GB", "GR", "GT", "HK", "HN", "HU", "ID",
- "IE", "IS", "IT", "JP", "LI", "LT", "LU", "LV", "MC", "MT",
- "MX", "MY", "NI", "NL", "NO", "NZ", "PA", "PE", "PH", "PL",
- "PT", "PY", "SE", "SG", "SK", "SV", "TR", "TW", "US", "UY"
- )
- val va: Artist = {
- val ar = new Artist()
- ar.setId("0LyfQWJT6nXafLPZqxe9Of")
- ar.setName("Various Artists")
- ar.setFollowers({
- val f = new Followers()
- f.setHref(null)
- f.setTotal(215138)
- f
- })
- ar.setExternalUrls({
- val e = new ExternalUrls()
- e.getExternalUrls.put("spotify",
- "https://open.spotify.com/artist/" + ar.getId)
- e
- })
- ar.setGenres(Seq.empty.asJava)
- ar.setHref("https://api.spotify.com/v1/artists/" + ar.getId)
- ar.setImages(Seq.empty.asJava)
- ar.setPopularity(0)
- ar.setUri("spotify:artist:" + ar.getId)
- ar
- }
- def simplify(a: Album): SimpleAlbum = {
- val sa = new SimpleAlbum()
- sa.setAlbumType(a.getAlbumType)
- sa.setExternalUrls(a.getExternalUrls)
- sa.setHref(a.getHref)
- sa.setId(a.getId)
- sa.setImages(a.getImages)
- sa.setName(a.getName)
- sa.setType(a.getType)
- sa.setUri(a.getUri)
- sa.setAvailableMarkets(a.getAvailableMarkets)
- sa
- }
- def simplify(t: Track): SimpleTrack = {
- val sa = new SimpleTrack()
- sa.setDiscNumber(t.getDiscNumber)
- sa.setDuration(t.getDuration)
- sa.setExplicit(t.isExplicit)
- sa.setExternalUrls(t.getExternalUrls)
- sa.setHref(t.getHref)
- sa.setId(t.getId)
- sa.setName(t.getName)
- sa.setPreviewUrl(t.getPreviewUrl)
- sa.setTrackNumber(t.getTrackNumber)
- sa.setType(t.getType)
- sa.setUri(t.getUri)
- sa.setAvailableMarkets(t.getAvailableMarkets)
- sa
- }
- def simplify(a: Artist): SimpleArtist = {
- val sa = new SimpleArtist()
- sa.setExternalUrls(a.getExternalUrls)
- sa.setHref(a.getHref)
- sa.setId(a.getId)
- sa.setName(a.getName)
- sa.setType(a.getType)
- sa.setUri(a.getUri)
- sa
- }
- /**
- * Select random elements from base list.
- * Must be count <= base.size()
- */
- def listFrom[T](ru: RandomUtil,
- base: Seq[T],
- count: Int): Seq[T] = {
- val n = base.size
- val ts = base.toBuffer
- // remove random elements from there
- for (i ← 0 to (n - count)) {
- ts.remove(ru.nextInt(n - i))
- }
- ts
- }
- def generateArtist(ru: RandomUtil): Artist = {
- val a = new Artist()
- a.setId(ru.string(8))
- a.setName("artist_" + a.getId)
- a.setPopularity(ru.nextInt(100))
- a.setExternalUrls({
- val eu = new ExternalUrls()
- eu.getExternalUrls.put("spotify",
- "https://open.spotify.com/artist/" + a.getId)
- eu
- })
- a.setHref("https://api.spotify.com/v1/artists/" + a.getId)
- a.setUri("spotify:artist:" + a.getId)
- a.setGenres(listFrom(ru, genres, 3).asJava)
- a.setFollowers({
- val f = new Followers()
- f.setTotal(ru.nextInt(1000))
- f
- })
- a
- }
- def generateAlbum(ru: RandomUtil,
- albumType: AlbumType,
- creators: Seq[Artist]): Album = {
- val al = new Album()
- al.setId(ru.string(9))
- al.setName("album_" + al.getId)
- al.setAlbumType(albumType)
- al.setHref("https://api.spotify.com/v1/albums/" + al.getId)
- al.setUri("spotify:album:" + al.getId)
- al.setCopyrights({
- val c1 = new Copyright()
- c1.setText("2016 Alpha Matrix")
- c1.setText("C")
- val c2 = new Copyright()
- c2.setText("2016 Alpha Matrix")
- c2.setText("P")
- Seq(c1, c2).asJava
- })
- al.setArtists(creators.map(simplify).asJava)
- al
- }
- def generateTrack(ru: RandomUtil,
- album: Album,
- trackNumber: Int): Track = {
- val t = new Track()
- t.setId(ru.string(10))
- t.setName("track_" + t.getId)
- t.setHref("https://api.spotify.com/v1/tracks/" + t.getId)
- t.setUri("spotify:track:" + t.getId)
- t.setArtists(album.getArtists)
- t.setAlbum(simplify(album))
- t.setTrackNumber(trackNumber)
- // from 1 to 1000 second
- t.setDuration((ru.nextInt(1000) + 1) * 1000)
- t.setPopularity(ru.nextInt(10))
- t.setAvailableMarkets(album.getAvailableMarkets)
- t.setDiscNumber(0)
- t.setPreviewUrl("https://spotify.com/preview/" + t.getId)
- t
- }
- def randomTask(ru: RandomUtil): CrawlerTask = {
- // set only entityType and externalId,
- // the rest of them should be populated by
- val ct = new CrawlerTask()
- ct.setEntityType(EntityType.fromInt(ru.nextInt(11, 13)))
- ct.setExternalId(ru.string(10))
- ct.setProvider(Provider.Spotify)
- ct
- }
- def randomTasks(ru: RandomUtil,
- sws: SpotifyWSImpl,
- count: Int): Seq[CrawlerTask] = {
- val keys1 = sws.artistMap.keys.toBuffer
- val keys2 = sws.albumMap.keys.toBuffer
- val keys3 = sws.trackMap.keys.toBuffer
- val n1 = keys1.size
- val n2 = keys2.size
- val n3 = keys3.size
- (1 to count).map { _ ⇒
- val n: Int = ru.nextInt(n1 + n2 + n3)
- val (i, et) =
- if (n < n1) (n, EntityType.Artist)
- else if (n1 <= n && n < n2) (n - n1, EntityType.Album)
- else (n - n1 - n2, EntityType.Song)
- val eid = et match {
- case EntityType.Artist ⇒ sws.artistMap(keys1(i)).getId
- case EntityType.Album ⇒ sws.albumMap(keys2(i)).getId
- case EntityType.Song ⇒ sws.trackMap(keys3(i)).getId
- }
- val ct = new CrawlerTask()
- ct.setProvider(Provider.Spotify)
- ct.setExternalId(eid)
- ct.setEntityType(et)
- ct
- }
- }
- def ids(cts: Seq[CrawlerTask]): Seq[String] =
- cts.map(_.getExternalId)
- trait SpotifyWS {
- def getArtists(ids: Seq[String]): Seq[Artist]
- def getAlbums(ids: Seq[String]): Seq[Album]
- def getTracks(ids: Seq[String]): Seq[Track]
- }
- class RxWrapper(sws: SpotifyWS) {
- def obsArtists(ids: Seq[String]): Observable[Artist] = Observable.from(sws.getArtists(ids))
- def obsAlbums(ids: Seq[String]): Observable[Album] = Observable.from(sws.getAlbums(ids))
- def obsTracks(ids: Seq[String]): Observable[Track] = Observable.from(sws.getTracks(ids))
- }
- class SpotifyWSImpl(ru: RandomUtil) extends SpotifyWS {
- val trackMap = new m.HashMap[String, Track]()
- val albumMap = new m.HashMap[String, Album]()
- val artistMap = new m.HashMap[String, Artist]()
- // go
- initMediaCollection()
- override def getArtists(ids: Seq[String]): Seq[Artist] = {
- val x = ru.nextInt(100)
- if (x < 10)
- throw new RuntimeException("timeout:" + Integer.toString(x))
- else
- ids.map(artistMap.getOrElse(_, null))
- }
- override def getAlbums(ids: Seq[String]): Seq[Album] = {
- val x = ru.nextInt(100)
- if (x < 10)
- throw new RuntimeException("timeout:" + Integer.toString(x))
- else
- ids.map(albumMap.getOrElse(_, null))
- }
- override def getTracks(ids: Seq[String]): Seq[Track] = {
- val x = ru.nextInt(100)
- if (x < 10)
- throw new RuntimeException("timeout:" + Integer.toString(x))
- else
- ids.map(trackMap.getOrElse(_, null))
- }
- def initMediaCollection(): Unit = {
- // create all
- val artists = (0 until 100).map(_ ⇒ generateArtist(ru))
- // let each artist to have 5 albums
- // 3 exclusive, 1 with other,
- // 1 with other two
- // and many album-compilations where
- // song is made by different artist
- // e.g. album 3m6QKbHMF5GVBagWlCF1U0
- val albums1 = artists.indices.flatMap({ i ⇒
- val ar = artists(i)
- val ars = Seq(ar)
- artistMap.put(ar.getId, ar)
- val al1 = generateAlbum(ru, AlbumType.SINGLE, ars)
- val al2 = generateAlbum(ru, AlbumType.SINGLE, ars)
- val al3 = generateAlbum(ru, AlbumType.SINGLE, ars)
- Seq(al1, al2, al3)
- }).toBuffer
- util.Random.shuffle(artists)
- val albums2 = artists.view.indices.drop(1).map({ i ⇒
- val ars = Seq(artists(i - 1), artists(i))
- generateAlbum(ru, AlbumType.ALBUM, ars)
- })
- // create albums of triples
- util.Random.shuffle(artists)
- val albums3 = artists.view.indices.drop(2).map({ i ⇒
- val ars = Seq(
- artists(i - 2),
- artists(i - 1),
- artists(i)
- )
- generateAlbum(ru, AlbumType.ALBUM, ars)
- })
- // create compilations,
- // albums where the artist is "Various Artist"
- val albums4 = (0 until 200).view.map({ i ⇒
- generateAlbum(ru, AlbumType.COMPILATION, Seq(va))
- })
- // glue all the stuff together
- val albums = albums1 ++ albums2 ++ albums3 ++ albums4
- // now create tracks, let each album has 10 tracks
- val tracks = albums.flatMap(al ⇒
- (1 to 10).map(generateTrack(ru, al, _)))
- // now put the stuff into the maps
- artists.foreach(ar ⇒ artistMap.put(ar.getId, ar))
- albums.foreach(al ⇒ albumMap.put(al.getId, al))
- tracks.foreach(t ⇒ trackMap.put(t.getId, t))
- }
- }
- def run(): Unit = {
- val sws = new SpotifyWSImpl(ru)
- val rxw = new RxWrapper(sws)
- val tasks = randomTasks(ru, sws, 10)
- Observable.from(tasks)
- .filter(ct ⇒ ct.getEntityType == EntityType.Song)
- .tumblingBuffer(2)
- .flatMap(cts ⇒ rxw.obsTracks(ids(cts)))
- .flatMap(t ⇒ {
- // we got track, now we want the artists and album from it
- // val artists = t.getArtists
- // val album = t.getAlbum
- Observable.just(t)
- })
- .subscribe(t ⇒ println(t))
- }
- run()
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement