From 235365e272e4b6900bbb3863596e2f2aadbfe248 Mon Sep 17 00:00:00 2001
From: Alex <ignatenko2207@gmail.com>
Date: Thu, 5 Nov 2020 01:50:01 +0200
Subject: [PATCH 1/2] Add SnagajobScraper, add method to load page with
 parameters and headers (it was necessary to get decoded data)

---
 .../scraper/webscraper/AWebScraper.groovy     |  62 +++++
 .../scraper/webscraper/SnagajobScraper.groovy | 227 ++++++++++++++++++
 2 files changed, 289 insertions(+)
 create mode 100644 src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy

diff --git a/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy b/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy
index 75695f6..8994922 100644
--- a/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy
+++ b/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy
@@ -128,6 +128,68 @@ abstract class AWebScraper extends AScraper {
 		return null
 	}
 
+	/** For loading a URL with parameters and headers to get a parsed Document **/
+	// NOTE: use this methd to load page in case if previous implementation returns encoded page
+	def loadPageWithParameters = { String url, Map params ->
+		try {
+			if (Thread.currentThread().isInterrupted()) return null	// Needed to prevent "MongoInterruptedException"???
+			def response = loadResponseWithParameters(url, params)
+			def parsedDocument = response?.parse()
+			if (response.statusCode() != 200) {
+				sleep 3 * 1000 // short wait to take pressure of the target server
+				response = loadResponse(url) // retry
+				if (response.statusCode() == 200) return response?.parse()
+				log.warn "StatusCode ${response.statusCode()} for page at URL $url"
+				this.cookiesForThread."${Thread.currentThread().getId()}" = [:] // reset cookies
+				return null
+			}
+			return parsedDocument
+		} catch (java.lang.OutOfMemoryError e) {
+			log.warn "$e for URL ${url}"
+			e.printStackTrace()
+		} catch (IllegalArgumentException e) { // url is null
+			log.warn "$e for URL ${url}"
+		} catch (HttpStatusException e) {
+			if (e.statusCode == 410) log.debug "HTTP Status 410 - URL is offline: ${url}"
+			else log.warn "$e #1 for $url"
+		} catch (java.io.IOException e) {
+			if (e.toString().contains("Mark invalid")) log.debug "Mark invalid - URL is offline: ${url}"
+			else log.warn "$e #2 for $url"
+		} catch (ConnectException e) {
+			log.warn "$e for URL ${url}"
+		} catch (UnknownHostException e) {
+			log.warn "$e for URL ${url} - Sleep for 0"
+		} catch (e) {
+			log.warn "$e for URL ${url} - Sleep for 0"
+		}
+		return null
+	}
+
+	def loadResponseWithParameters = { String url, Map params ->
+		if (Thread.currentThread().isInterrupted()) return null	// Needed to prevent "MongoInterruptedException"???
+		def cookies = this.cookiesForThread."${Thread.currentThread().getId()}" ?: [:]
+		if (params.urlParams) {
+			if (url.contains("?")) {
+				url += "&"
+			} else {
+				url += "?"
+			}
+			url += (params.urlParams as Map).entrySet().join("&")
+		}
+		def response = Jsoup.connect(url)
+				.timeout(TIMEOUT)
+				.userAgent(USER_AGENT)
+				.followRedirects(true)
+				.ignoreContentType(true)
+				.ignoreHttpErrors(true)
+				.headers(params.headers ?: [:])
+		if (!cookies) response = response.referrer("https://www.google.com/")
+		response = response.execute()
+		def newCookies = response.cookies()
+		if (newCookies) this.cookiesForThread."${Thread.currentThread().getId()}" = (cookies ?: [:]) + newCookies
+		return response
+	}
+
 	/** For loading JSON, XML, etc. **/
 	def loadParseable = { url, payload ->
 		try {
diff --git a/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy b/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy
new file mode 100644
index 0000000..56c316a
--- /dev/null
+++ b/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy
@@ -0,0 +1,227 @@
+/* Copyright © 2020, TechMap GmbH - All rights reserved. */
+package io.techmap.scrape.scraper.webscraper
+
+import groovy.json.JsonSlurper
+import groovy.time.TimeCategory
+import groovy.util.logging.Log4j2
+import io.techmap.scrape.data.Company
+import io.techmap.scrape.data.Job
+import io.techmap.scrape.data.Location
+import io.techmap.scrape.data.shared.TagType
+import io.techmap.scrape.helpers.DataCleaner
+import org.bson.Document
+import org.jsoup.HttpStatusException
+import org.jsoup.nodes.Element
+import org.jsoup.select.Elements
+
+import java.time.LocalDate
+import java.time.ZonedDateTime
+
+@Log4j2
+class SnagajobScraper extends AWebScraper {
+
+    /**
+     * List of websites with different TLDs but the same page structure
+     * Normally the oldest source will be selected - in this test environment only the first is used
+     **/
+    static final ArrayList sources = [
+            [id: "us", url: "https://www.snagajob.com"]
+    ]
+    static final String baseSourceID = 'snagajob_'
+
+    SnagajobScraper(Integer sourceToScrape) {
+        super(sources, baseSourceID)
+        this.sourceToScrape = sourceToScrape
+        this.source = this.sources[sourceToScrape]
+        this.sourceID = this.baseSourceID + this.source.id
+        log.info "Using userAgent: ${USER_AGENT}"
+    }
+
+    @Override
+    int scrape() {
+        super.initScrape()
+
+        Integer jobsInSourceCount = 0
+
+        // NOTE: to load page correctly do not use old method, it returns encoded page that cannot be parsed
+        def startPage = loadPageWithParameters("${source.url}", [:])
+        final def startCookies = this.cookiesForThread."${Thread.currentThread().getId()}" ?: [:]
+
+        // The first page doesn't contain groups
+        final JsonSlurper jsonSlurper = new JsonSlurper()
+        def searchPageScript = startPage.select("script[type=application/ld+json]")?.find({ it.html().contains("SearchAction") })?.html()
+        def searchData = jsonSlurper.parseText(searchPageScript)
+        def searchLink = (searchData?.potentialAction?.target as String)?.split("/q-")?.first()
+        def searchPage = loadPageWithParameters(searchLink, [:])
+
+        // Identify groups of jobs such as categories, industries or Jobnames we can iterate over
+        def parametersScript = searchPage.select("script#seeker-state")?.html()?.replaceAll("&q;", "\"")?.replaceAll("&a;", "&")
+        def parametersData = jsonSlurper.parseText(parametersScript)
+        def groupSearchParams = parametersData?.values()?.find({ it.body.filterGroups })?.body?.filterGroups?.find({ it.name == "Industry" })
+        def groups = groupSearchParams?.list?.sort { it?.name } // sort necessary for compare with status.lastCategory
+
+        for (Map group in groups) {
+            def status = db.loadStatus(sourceID + "-${group.name}")
+            this.cookiesForThread."${Thread.currentThread().getId()}" = startCookies
+            // this resource cannot be parsed by old way
+            // there are a lot of scripts and json data(with search parameters) to get next links and content
+            group += [  searchLink: searchLink,
+                        urlParams: [
+                            "${groupSearchParams.value}" : group.value]]
+            int jobsInCategoryCount = scrapePageGroupWithParameters(group, status)
+            jobsInSourceCount += jobsInCategoryCount
+            if (maxDocsToPrint <= 0) break
+        }
+        return jobsInSourceCount
+    }
+
+    @Override
+    int scrapePageGroup(Element group, Map status) {
+        return 0
+    }
+
+    int scrapePageGroupWithParameters(Map group, Map status) {
+        def startTime = new Date()
+        log.debug "... starting to scrape group ${group.name}"
+
+        def nextURL         = group.searchLink
+        def paginationPage  = loadPageWithParameters(nextURL, [urlParams: group.urlParams])
+
+        int maxJobsInGroup = (paginationPage?.select("h1.results-heading")?.text()?.replaceAll("\\D", "") ?: 0)?.toInteger()
+
+        int offset = 0
+        int jobsInGroupCount = 0
+        while (nextURL) {
+            // NOTE: this mechanism ready the "next" URL in a pagination - some pages might require different approaches (e.g., increasing a "page" parameter)
+            def jobLinks = paginationPage?.select("job-overview[itemprop=itemListElement] meta[itemprop=url]")
+            int jobsInJobListCount = scrapePageList(jobLinks, [category: group.name])
+            jobsInGroupCount += jobsInJobListCount
+            log.debug "... scraped ${"$jobsInJobListCount".padLeft(4)} of ${"$maxJobsInGroup".padLeft(5)} jobs with offset $offset in group ${group.text()}"
+
+            // if (jobsInJobListCount <= 0) break // switch deep scraping (when disabled) and shallow scraping (when enabled)
+            if (maxDocsToPrint <= 0) break
+
+            // Get next URL and load page for next iteration
+            offset = Math.max(status.lastOffset ?: 0, jobLinks.size() ?: 15)?.toInteger()
+            nextURL = paginationPage?.select("a:contains(Next)")?.first()?.absUrl("href")
+            if (nextURL) {
+                paginationPage = loadPageWithParameters(nextURL, [urlParams: group.urlParams])
+                status.lastOffset = offset
+                db.saveStatus(status)
+            }
+        }
+        status.lastOffset = 0 // Reset
+        db.saveStatus(status)
+        log.info "Scraped ${"$jobsInGroupCount".padLeft(5)} of ${"$maxJobsInGroup".padLeft(6)} jobs in group ${group.text()} in " + TimeCategory.minus(new Date(), startTime)
+        return jobsInGroupCount
+    }
+
+    @Override
+    int scrapePageList(Elements pageElements, Map extraData) {
+        int jobsInPageCount = 0
+        for (pageElement in pageElements) {
+            String jobPageURL = pageElement?.absUrl("content")
+            String idInSource = jobPageURL?.split("/")?.last()
+            if (!db.jobExists(sourceID, idInSource)) {
+                extraData.idInSource = idInSource
+                if (scrapePage(jobPageURL, extraData)) jobsInPageCount++
+            }
+            if (maxDocsToPrint <= 0) break
+        }
+        return jobsInPageCount
+    }
+
+    // @formatter:off (to keep the code in a more tabular form - "Align variables in columns" only works for class fields )
+    @Override
+    boolean scrapePage(String pageURL, Map extraData) {
+        try {
+            def jobPage = loadPageWithParameters(pageURL, [:])
+            if (!jobPage) return false
+
+            /*******************************/
+            /* Extract data in JSON format */
+            /*******************************/
+
+            final JsonSlurper jsonSlurper = new JsonSlurper()
+            def dataRaw = jobPage?.select("script")?.find({it.html().contains("JobPosting")})?.html()
+            def data = jsonSlurper.parseText(dataRaw ?: "{}")
+
+            /*****************/
+            /* Fill Job data */
+            /*****************/
+
+            Job job         = new Job()
+            job.source      = sourceID
+            job.idInSource  = extraData.idInSource ?: jobPage.select("div.posting-details span[class=ng-star-inserted]")?.first()?.text()?.replaceAll("\\D", "")
+            job.url         = pageURL ?: data?.url
+            job.name        = data?.title ?: jobPage.select("h2.h1")?.first()?.text()
+
+            job.html = jobPage?.select("job-details")?.first()?.html()
+            job.text = DataCleaner.stripHTML(job.html)
+            job.json = [:]
+            if (data) job.json.pageData = data
+
+            try {
+                job.dateCreated = ZonedDateTime.parse(data?.datePosted)?.toLocalDateTime()
+                job.dateCreated = job.dateCreated ?: LocalDate.parse(jobPage.select("div.posting-details span.divider")?.last()?.text()?.replaceAll("\\D", ""), "yyyyMMdd").atStartOfDay()
+            } catch (e) { /*ignore*/ }
+
+            job.position.name           = job.name
+            job.position.workType       = jobPage.select("job-details i.icon-time")?.first()?.parent()?.text() ?: data?.employmentType
+
+            job.salary.text             = jobPage.select("#estimatedWage")?.first()?.text()
+            job.salary.currency         = data?.estimatedSalary?.get(0)?.currency
+            job.salary.value            = data?.estimatedSalary?.get(0)?.minValue
+            job.salary.value            = job.salary.value ?: data?.estimatedSalary?.get(0)?.maxValue
+            job.salary.period           = data?.estimatedSalary?.get(0)?.unitText
+
+            job.orgTags."${TagType.CATEGORIES}" = (job.orgTags."${TagType.CATEGORIES}" ?: []) + extraData?.category
+
+            /**********************/
+            /* Fill Location data */
+            /**********************/
+
+            Location location = new Location()
+            location.source = sourceID
+            location.orgAddress.addressLine     = jobPage.select("job-details i.icon-location")?.first()?.parent()?.parent()?.text()
+            location.orgAddress.countryCode     = data?.jobLocation?.address?.addressCountry?.name
+            location.orgAddress.state           = data?.jobLocation?.address?.addressRegion
+            location.orgAddress.city            = data?.jobLocation?.address?.addressLocality
+            location.orgAddress.street          = data?.jobLocation?.address?.streetAddress
+            location.orgAddress.postCode        = data?.jobLocation?.address?.postalCode
+            location.orgAddress.geoPoint.lat    = data?.jobLocation?.geo?.latitude as Double
+            location.orgAddress.geoPoint.lng    = data?.jobLocation?.geo?.longitude as Double
+
+            /*********************/
+            /* Fill Company data */
+            /*********************/
+
+            Company company     = new Company()
+            company.source      = sourceID
+            company.name        = data?.hiringOrganization?.name ?: jobPage.select("job-details .company-name")?.first()?.text()
+
+            company.urls = [("$sourceID" as String): companyLink]
+            company.ids  = [("$sourceID" as String): company.idInSource] // no company id was found
+
+            /*******************/
+            /* Store page data */
+            /*******************/
+
+            Document rawPage = new Document()
+            rawPage.url = job.url
+            rawPage.html = jobPage.html()
+
+            return crossreferenceAndSaveData(job, location, company, rawPage)
+        } catch (HttpStatusException e) {
+            log.error "$e for $pageURL"
+        } catch (IOException e) {
+            log.error "$e for $pageURL"
+        } catch (NullPointerException e) {
+            log.error "$e for $pageURL" // probably a problem with SimpleDateFormat (do not store job)
+        } catch (e) {
+            log.error "$e for $pageURL"
+            e.printStackTrace()
+        }
+        return false
+    }
+}

From 105dc642b7b3a43264759ef35c7316150b85b6bd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jo=CC=88rg=20Rech?= <joerg.rech@gmail.com>
Date: Wed, 11 Nov 2020 11:40:22 +0100
Subject: [PATCH 2/2] 1. your workaround for the compression was not really
 necessary - you could just have disabled the "Accept-Encoding" header (Some
 Webserver handle them incorrectly or use strange compression algorithms). 2.
 The scraper did not work at first as you missed to define the "companyLink"
 3. You do not always have to start at the homepage and drill to a
 category-page - you can just start there (e.g., at
 https://www.snagajob.com/jobs-by/industry). 4. You missed the second JSON
 data package - which is included on every page (the schema.org JobPosting
 data package is not available always). This JSON contains very interesting
 info such as street, industry, etc. 5. Parsing of dateCreated did not work if
 the first line with "ZonedDateTime.parse()" threw an Exception

---
 .../scraper/webscraper/AWebScraper.groovy     |   2 +-
 .../scraper/webscraper/SnagajobScraper.groovy | 121 +++++++++---------
 2 files changed, 59 insertions(+), 64 deletions(-)

diff --git a/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy b/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy
index 8994922..688a402 100644
--- a/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy
+++ b/src/main/groovy/io/techmap/scrape/scraper/webscraper/AWebScraper.groovy
@@ -58,7 +58,7 @@ abstract class AWebScraper extends AScraper {
 				.timeout(TIMEOUT)
 				.userAgent(USER_AGENT)
 				.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
-				.header("Accept-Encoding", "gzip, deflate, br")
+//				.header("Accept-Encoding", "gzip, deflate, br")
 				.header("Accept-Language", "en-US,en;q=0.9,de;q=0.8,es;q=0.7")
 				.header("Cache-Control", "no-cache") // for Monster.at
 				.header("Connection", "keep-alive") // for Monster.at
diff --git a/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy b/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy
index 56c316a..822b67e 100644
--- a/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy
+++ b/src/main/groovy/io/techmap/scrape/scraper/webscraper/SnagajobScraper.groovy
@@ -25,7 +25,7 @@ class SnagajobScraper extends AWebScraper {
      * Normally the oldest source will be selected - in this test environment only the first is used
      **/
     static final ArrayList sources = [
-            [id: "us", url: "https://www.snagajob.com"]
+            [id: "us", url: "https://www.snagajob.com/jobs-by/industry"]
     ]
     static final String baseSourceID = 'snagajob_'
 
@@ -43,33 +43,16 @@ class SnagajobScraper extends AWebScraper {
 
         Integer jobsInSourceCount = 0
 
-        // NOTE: to load page correctly do not use old method, it returns encoded page that cannot be parsed
-        def startPage = loadPageWithParameters("${source.url}", [:])
+		def startPage = loadPage("${source.url}")
         final def startCookies = this.cookiesForThread."${Thread.currentThread().getId()}" ?: [:]
-
-        // The first page doesn't contain groups
-        final JsonSlurper jsonSlurper = new JsonSlurper()
-        def searchPageScript = startPage.select("script[type=application/ld+json]")?.find({ it.html().contains("SearchAction") })?.html()
-        def searchData = jsonSlurper.parseText(searchPageScript)
-        def searchLink = (searchData?.potentialAction?.target as String)?.split("/q-")?.first()
-        def searchPage = loadPageWithParameters(searchLink, [:])
-
-        // Identify groups of jobs such as categories, industries or Jobnames we can iterate over
-        def parametersScript = searchPage.select("script#seeker-state")?.html()?.replaceAll("&q;", "\"")?.replaceAll("&a;", "&")
-        def parametersData = jsonSlurper.parseText(parametersScript)
-        def groupSearchParams = parametersData?.values()?.find({ it.body.filterGroups })?.body?.filterGroups?.find({ it.name == "Industry" })
-        def groups = groupSearchParams?.list?.sort { it?.name } // sort necessary for compare with status.lastCategory
-
-        for (Map group in groups) {
-            def status = db.loadStatus(sourceID + "-${group.name}")
-            this.cookiesForThread."${Thread.currentThread().getId()}" = startCookies
-            // this resource cannot be parsed by old way
-            // there are a lot of scripts and json data(with search parameters) to get next links and content
-            group += [  searchLink: searchLink,
-                        urlParams: [
-                            "${groupSearchParams.value}" : group.value]]
-            int jobsInCategoryCount = scrapePageGroupWithParameters(group, status)
-            jobsInSourceCount += jobsInCategoryCount
+	
+		def groups = startPage.select(".industries > div > a")?.sort { it.text() } // sort necessary for compare with status.lastCategory
+
+        for (group in groups) {
+			def status = db.loadStatus(sourceID + "-${group.text()}")
+			this.cookiesForThread."${Thread.currentThread().getId()}" = startCookies
+			int jobsInCategoryCount = scrapePageGroup(group, status)
+			jobsInSourceCount += jobsInCategoryCount
             if (maxDocsToPrint <= 0) break
         }
         return jobsInSourceCount
@@ -77,15 +60,11 @@ class SnagajobScraper extends AWebScraper {
 
     @Override
     int scrapePageGroup(Element group, Map status) {
-        return 0
-    }
-
-    int scrapePageGroupWithParameters(Map group, Map status) {
         def startTime = new Date()
-        log.debug "... starting to scrape group ${group.name}"
-
-        def nextURL         = group.searchLink
-        def paginationPage  = loadPageWithParameters(nextURL, [urlParams: group.urlParams])
+        log.debug "... starting to scrape group ${group.text()}"
+	
+		def nextURL = group.absUrl("href")
+		def paginationPage = loadPage(nextURL)
 
         int maxJobsInGroup = (paginationPage?.select("h1.results-heading")?.text()?.replaceAll("\\D", "") ?: 0)?.toInteger()
 
@@ -94,7 +73,7 @@ class SnagajobScraper extends AWebScraper {
         while (nextURL) {
             // NOTE: this mechanism ready the "next" URL in a pagination - some pages might require different approaches (e.g., increasing a "page" parameter)
             def jobLinks = paginationPage?.select("job-overview[itemprop=itemListElement] meta[itemprop=url]")
-            int jobsInJobListCount = scrapePageList(jobLinks, [category: group.name])
+            int jobsInJobListCount = scrapePageList(jobLinks, [category: group.text()])
             jobsInGroupCount += jobsInJobListCount
             log.debug "... scraped ${"$jobsInJobListCount".padLeft(4)} of ${"$maxJobsInGroup".padLeft(5)} jobs with offset $offset in group ${group.text()}"
 
@@ -103,9 +82,9 @@ class SnagajobScraper extends AWebScraper {
 
             // Get next URL and load page for next iteration
             offset = Math.max(status.lastOffset ?: 0, jobLinks.size() ?: 15)?.toInteger()
-            nextURL = paginationPage?.select("a:contains(Next)")?.first()?.absUrl("href")
+            nextURL = paginationPage?.select("noscript a:Contains(Next)")?.first()?.absUrl("href")
             if (nextURL) {
-                paginationPage = loadPageWithParameters(nextURL, [urlParams: group.urlParams])
+                paginationPage = loadPage(nextURL)
                 status.lastOffset = offset
                 db.saveStatus(status)
             }
@@ -135,7 +114,7 @@ class SnagajobScraper extends AWebScraper {
     @Override
     boolean scrapePage(String pageURL, Map extraData) {
         try {
-            def jobPage = loadPageWithParameters(pageURL, [:])
+            def jobPage = loadPage(pageURL)
             if (!jobPage) return false
 
             /*******************************/
@@ -145,7 +124,12 @@ class SnagajobScraper extends AWebScraper {
             final JsonSlurper jsonSlurper = new JsonSlurper()
             def dataRaw = jobPage?.select("script")?.find({it.html().contains("JobPosting")})?.html()
             def data = jsonSlurper.parseText(dataRaw ?: "{}")
-
+			def dataRaw2 = jobPage?.select("script#seeker-state")?.first()?.html()?.replaceAll(/\&q;/,'"')
+			def data2 = jsonSlurper.parseText(dataRaw2 ?: "{}").entrySet().iterator().next().getValue().body
+			if (data2?.isHoneypot) {
+				sleep 1	// WARN: looks like this is a Honeypot to identify scrapers - but the flag is only visible after loading and snagajob blocks after ~20 pages (but unblocks after 19 blocka ???)
+			}
+			
             /*****************/
             /* Fill Job data */
             /*****************/
@@ -154,28 +138,37 @@ class SnagajobScraper extends AWebScraper {
             job.source      = sourceID
             job.idInSource  = extraData.idInSource ?: jobPage.select("div.posting-details span[class=ng-star-inserted]")?.first()?.text()?.replaceAll("\\D", "")
             job.url         = pageURL ?: data?.url
-            job.name        = data?.title ?: jobPage.select("h2.h1")?.first()?.text()
+            job.name        = data?.title ?: data2?.title ?: jobPage.select("h2.h1")?.first()?.text()
 
-            job.html = jobPage?.select("job-details")?.first()?.html()
+            job.html = jobPage?.select("jobs-description")?.first()?.html() ?: data2.description ?: data2.postingOverview
             job.text = DataCleaner.stripHTML(job.html)
             job.json = [:]
-            if (data) job.json.pageData = data
-
-            try {
-                job.dateCreated = ZonedDateTime.parse(data?.datePosted)?.toLocalDateTime()
-                job.dateCreated = job.dateCreated ?: LocalDate.parse(jobPage.select("div.posting-details span.divider")?.last()?.text()?.replaceAll("\\D", ""), "yyyyMMdd").atStartOfDay()
-            } catch (e) { /*ignore*/ }
+            if (data)	job.json.schemaOrg	= data
+			if (data2)	job.json.pageData	= data2
+			
+            try { job.dateCreated = ZonedDateTime.parse(data?.datePosted)?.toLocalDateTime() } catch (e) { /*ignore*/ }
+			try { job.dateCreated = job.dateCreated ?: ZonedDateTime.parse(data2?.internalCreateDate)?.toLocalDateTime()} catch (e) { /*ignore*/ }
+			try { job.dateCreated = job.dateCreated ?: LocalDate.parse(jobPage.select("div.posting-details span.divider")?.last()?.text()?.replaceAll("\\D", ""), "yyyyMMdd").atStartOfDay() } catch (e) { /*ignore*/ }
+			try { job.datesUpdated += ZonedDateTime.parse(data2?.internalUpdateDate)?.toLocalDateTime()} catch (e) { /*ignore*/ }
 
             job.position.name           = job.name
             job.position.workType       = jobPage.select("job-details i.icon-time")?.first()?.parent()?.text() ?: data?.employmentType
 
-            job.salary.text             = jobPage.select("#estimatedWage")?.first()?.text()
-            job.salary.currency         = data?.estimatedSalary?.get(0)?.currency
-            job.salary.value            = data?.estimatedSalary?.get(0)?.minValue
-            job.salary.value            = job.salary.value ?: data?.estimatedSalary?.get(0)?.maxValue
+            job.salary.text             = data2?.wage?.text // NOTE: estimates are too dangerous
+            job.salary.currency         = data2?.wage?.currency ?: data?.estimatedSalary?.get(0)?.currency
+            job.salary.value            = data2?.wage?.value	?: data2?.wage?.median
+			job.salary.value            = job.salary.value		?: data2?.wage?.minimum ?: data?.estimatedSalary?.get(0)?.minValue
+			job.salary.value            = job.salary.value		?: data2?.wage?.maximum ?: data?.estimatedSalary?.get(0)?.maxValue
             job.salary.period           = data?.estimatedSalary?.get(0)?.unitText
-
-            job.orgTags."${TagType.CATEGORIES}" = (job.orgTags."${TagType.CATEGORIES}" ?: []) + extraData?.category
+	
+			job.orgTags."${TagType.CATEGORIES}"	= (job.orgTags."${TagType.CATEGORIES}" ?: [])	+ extraData?.category
+			job.orgTags."${TagType.INDUSTRIES}"	= (job.orgTags."${TagType.INDUSTRIES}" ?: [])	+ data2?.primaryIndustryName
+			job.orgTags."${TagType.INDUSTRIES}"	= (job.orgTags."${TagType.INDUSTRIES}" ?: [])	+ data2?.industries
+			job.orgTags."${TagType.JOBNAMES}"	= (job.orgTags."${TagType.JOBNAMES}" ?: [])		+ data2?.titleNormalized
+			job.orgTags."${TagType.JOBNAMES}"	= (job.orgTags."${TagType.JOBNAMES}" ?: [])		+ data2?.seoJobTitle
+			job.orgTags."${TagType.COMPANY_BENEFITS}"	= (job.orgTags."${TagType.COMPANY_BENEFITS}" ?: [])		+ data2?.qualifications?.jobBenefits
+			job.orgTags."${TagType.QUALIFICATIONS}"		= (job.orgTags."${TagType.QUALIFICATIONS}" ?: [])		+ data2?.qualifications?.jobRequirements
+			job.orgTags."${TagType.KEYWORDS}"		= (job.orgTags."${TagType.KEYWORDS}" ?: [])		+ data2?.fextures*.name // NOTE: "fextures" is not a misspelling!
 
             /**********************/
             /* Fill Location data */
@@ -183,14 +176,15 @@ class SnagajobScraper extends AWebScraper {
 
             Location location = new Location()
             location.source = sourceID
-            location.orgAddress.addressLine     = jobPage.select("job-details i.icon-location")?.first()?.parent()?.parent()?.text()
+            location.orgAddress.addressLine     = jobPage.select("job-details i.icon-location")?.first()?.parent()?.parent()?.text() ?: data2?.location?.locationName
             location.orgAddress.countryCode     = data?.jobLocation?.address?.addressCountry?.name
-            location.orgAddress.state           = data?.jobLocation?.address?.addressRegion
-            location.orgAddress.city            = data?.jobLocation?.address?.addressLocality
+            location.orgAddress.state           = data?.jobLocation?.address?.addressRegion		?: data2?.location?.stateProvince ?: data2?.location?.stateProvinceCode
+            location.orgAddress.city            = data?.jobLocation?.address?.addressLocality	?: data2?.location?.city
             location.orgAddress.street          = data?.jobLocation?.address?.streetAddress
-            location.orgAddress.postCode        = data?.jobLocation?.address?.postalCode
-            location.orgAddress.geoPoint.lat    = data?.jobLocation?.geo?.latitude as Double
-            location.orgAddress.geoPoint.lng    = data?.jobLocation?.geo?.longitude as Double
+			location.orgAddress.postCode        = data?.jobLocation?.address?.postalCode		?: data2?.location?.postalCode
+			location.orgAddress.street        	= data2?.location?.addressLine1
+            location.orgAddress.geoPoint.lat    = (data?.jobLocation?.geo?.latitude				?: data2?.latitude)		as Double
+            location.orgAddress.geoPoint.lng    = (data?.jobLocation?.geo?.longitude			?: data2?.longitude)	as Double
 
             /*********************/
             /* Fill Company data */
@@ -198,10 +192,11 @@ class SnagajobScraper extends AWebScraper {
 
             Company company     = new Company()
             company.source      = sourceID
-            company.name        = data?.hiringOrganization?.name ?: jobPage.select("job-details .company-name")?.first()?.text()
-
+			company.url        	= data?.hiringOrganization?.url
+			company.name        = data?.hiringOrganization?.name ?: jobPage.select("job-details .company-name")?.first()?.text()
+			def companyLink = jobPage.select("job-details .company-name")?.first()?.parent()?.absUrl("href")
             company.urls = [("$sourceID" as String): companyLink]
-            company.ids  = [("$sourceID" as String): company.idInSource] // no company id was found
+            company.ids  = [("$sourceID" as String): companyLink?.replaceAll(/.*\/company\/([^\/\?]+)/,'$1')]
 
             /*******************/
             /* Store page data */
