I'm seeing this if I attempt to schedule an export, tested on 7.2.x and 7.0.5:
/usr/bin/phantomjs --ignore-ssl-errors=true /tmp/tmpB8poJh
------ Step : 0 (1550465709271) -------
waitpdf = false
loadInProgress = false
typeof steps[testindex] = function
Open login page : https://localhost:8000/en-US/account/login
------ Step : 1 (1550465729271) -------
waitpdf = false
loadInProgress = false
typeof steps[testindex] = function
authenticating...
0
------ Step : 2 (1550465749271) -------
waitpdf = false
loadInProgress = false
typeof steps[testindex] = function
Open dashboard : https://localhost:8000/en-US/app/smart_exporter_app/panel_grouping_with_charts
------ Step : 3 (1550465770520) -------
waitpdf = false
loadInProgress = false
typeof steps[testindex] = function
Smart exporter exists ... Generating PDF
generate pdf clicked !
Canvas width and/or height not set. Defaults used.
0
Canvas width and/or height not set. Defaults used.
Canvas width and/or height not set. Defaults used.
Canvas width and/or height not set. Defaults used.
Canvas width and/or height not set. Defaults used.
Canvas width and/or height not set. Defaults used.
Unhandled promise rejection TypeError: undefined is not an object (evaluating '$(rows[curr]).find("svg").find("text").css("font-size").split')
https://localhost:8000/en-US/static/@be11b2c46e23/build/api/SplunkVisualizationUtils.js:59
------ Step : 4 (1550465790516) -------
waitpdf = true
loadInProgress = false
typeof steps[testindex] = function
Wait for PDF Export to finish ...
I'm running the scheduled export against the default panel_grouping_with_charts and it's failing, it goes into that loop forever.
phantomjs --version
2.1.1
So with some advice from @badarsebard on slack I managed to get this closer to working, but still not happy with the results!
smart_exporter_app\appserver\controllers\service.py
I changed:
#email_user = settings['email_user']
#email_pwd = settings['email_pwd']
email_user = ""
email_pwd = ""
#if email_user in [None, '']:
# raise Exception('Mail Server Credentials should be setup.')
#if email_pwd in [None, '']:
# raise Exception('Mail Server Credentials should be setup.')
(note the above is in two places in the file)
Also
#srv.starttls()
As my mail server does not have TLS support or credentials.
static/script.txt was replaced with:
const puppeteer = require('puppeteer');
const fs = require('fs');
const util = require('util');
const chunks = [];
const path = require('path');
(async () => {
// load login
console.log("launching browser");
const browser = await puppeteer.launch({headless: true, ignoreHTTPSErrors: true});
const page = await browser.newPage();
async function chunkPDF(chunkid, id) {
console.log("### chunk call (" + chunkid + "/" + chunks.length + ") ###");
if (chunkid == chunks.length - 1) {
await page.goto("https://localhost:8000/en-US/custom/smart_exporter_app/service/sendpdfexport/" + encodeURIComponent(chunks[chunkid]) + "/" + id.toString() + "/" + chunkid.toString() + "/" + chunks.length.toString() + "/" + ("##scheduleparams##").split(".json").join(""));
await setTimeout(function () {
console.log("end recursive call ...");
}, 1000);
} else {
if (chunkid < chunks.length - 1) {
await page.goto("https://localhost:8000/en-US/custom/smart_exporter_app/service/sendpdfexport/" + encodeURIComponent(chunks[chunkid]) + "/" + id.toString() + "/" + chunkid.toString() + "/" + chunks.length.toString() + "/" + ("##scheduleparams##").split(".json").join(""));
chunkid = chunkid + 1;
await chunkPDF(chunkid, id);
}
}
}
console.log("opening login screen");
await page.goto('https://localhost:8000', {waitUntil: "networkidle0"});
// enter creds
const forms = await page.$$(".loginForm");
for (let i = 0; i < forms.length; i++) {
if (await forms[i].$("#username") && await forms[i].$("#password")) {
await page.type("#username", "##username##"); // ##username##
await page.type("#password", "##password##"); // ##password##
let button = await forms[i].$("input.btn");
await button.click();
// await page.evaluate(form => form.submit(), forms[i]);
break;
} else {
console.log('not login form');
}
}
await page.waitForNavigation();
// goto dash
console.log("opening dashboard ##dashbord_URL##");
await page.goto("##dashbord_URL##", {waitUntil: "networkidle0"}); // ##dashbord_URL##
await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: '/tmp/smartpdf'});
// click the button
if (fs.existsSync("/tmp/export.pdf")) {
await fs.unlinkSync("/tmp/export.pdf");
}
console.log("generate report");
const exportButton = await page.$("#generateExport");
await exportButton.click();
console.log("wait for download to finish");
await page.waitFor(1000);
let fileName;
while (!fileName || fileName.endsWith('.crdownload')) {
await new Promise(resolve => setTimeout(resolve, 1000));
[fileName] = await util.promisify(fs.readdir)('/tmp/smartpdf');
}
const filePath = path.resolve('/tmp/smartpdf', fileName);
console.error('Downloaded file:', filePath);
console.log("download finished");
// pick up file from /tmp
console.log("reading report");
const contents = fs.readFileSync(filePath);
const stream = contents.toString('base64');
// chunk file
for (let i = 0, charsLength = stream.length; i < charsLength; i += 50000) {
await chunks.push(stream.substring(i, i + 50000));
}
const id = (new Date).getTime();
await fs.unlinkSync(filePath);
console.log("recursive call to chunk pdf...");
await chunkPDF(0, id);
// closing
console.log("hard wait");
await page.waitFor(15000);
await browser.close();
})();
smart_exporter_app\bin\smartexporter.py
cmdargs = os.path.join(phantomjs_path,"phantomjs")+ " "+f.name
i also replaced:
/opt/phantomjs/bin/phantomjs
With a symlink to:
/opt/splunk/bin/node
And I installed npm and:
npm --proxy "http://proxy:8080" i puppeteer
So this swaps out phantomjs with puppeteer which is based on chromium, perhaps I just have an old version but the rendering is pretty terrible of the dashboards.
So not working well but definitely a lot better than it was, and perhaps with a modern version of puppeteer it might work better...
So with some advice from @badarsebard on slack I managed to get this closer to working, but still not happy with the results!
smart_exporter_app\appserver\controllers\service.py
I changed:
#email_user = settings['email_user']
#email_pwd = settings['email_pwd']
email_user = ""
email_pwd = ""
#if email_user in [None, '']:
# raise Exception('Mail Server Credentials should be setup.')
#if email_pwd in [None, '']:
# raise Exception('Mail Server Credentials should be setup.')
(note the above is in two places in the file)
Also
#srv.starttls()
As my mail server does not have TLS support or credentials.
static/script.txt was replaced with:
const puppeteer = require('puppeteer');
const fs = require('fs');
const util = require('util');
const chunks = [];
const path = require('path');
(async () => {
// load login
console.log("launching browser");
const browser = await puppeteer.launch({headless: true, ignoreHTTPSErrors: true});
const page = await browser.newPage();
async function chunkPDF(chunkid, id) {
console.log("### chunk call (" + chunkid + "/" + chunks.length + ") ###");
if (chunkid == chunks.length - 1) {
await page.goto("https://localhost:8000/en-US/custom/smart_exporter_app/service/sendpdfexport/" + encodeURIComponent(chunks[chunkid]) + "/" + id.toString() + "/" + chunkid.toString() + "/" + chunks.length.toString() + "/" + ("##scheduleparams##").split(".json").join(""));
await setTimeout(function () {
console.log("end recursive call ...");
}, 1000);
} else {
if (chunkid < chunks.length - 1) {
await page.goto("https://localhost:8000/en-US/custom/smart_exporter_app/service/sendpdfexport/" + encodeURIComponent(chunks[chunkid]) + "/" + id.toString() + "/" + chunkid.toString() + "/" + chunks.length.toString() + "/" + ("##scheduleparams##").split(".json").join(""));
chunkid = chunkid + 1;
await chunkPDF(chunkid, id);
}
}
}
console.log("opening login screen");
await page.goto('https://localhost:8000', {waitUntil: "networkidle0"});
// enter creds
const forms = await page.$$(".loginForm");
for (let i = 0; i < forms.length; i++) {
if (await forms[i].$("#username") && await forms[i].$("#password")) {
await page.type("#username", "##username##"); // ##username##
await page.type("#password", "##password##"); // ##password##
let button = await forms[i].$("input.btn");
await button.click();
// await page.evaluate(form => form.submit(), forms[i]);
break;
} else {
console.log('not login form');
}
}
await page.waitForNavigation();
// goto dash
console.log("opening dashboard ##dashbord_URL##");
await page.goto("##dashbord_URL##", {waitUntil: "networkidle0"}); // ##dashbord_URL##
await page._client.send('Page.setDownloadBehavior', {behavior: 'allow', downloadPath: '/tmp/smartpdf'});
// click the button
if (fs.existsSync("/tmp/export.pdf")) {
await fs.unlinkSync("/tmp/export.pdf");
}
console.log("generate report");
const exportButton = await page.$("#generateExport");
await exportButton.click();
console.log("wait for download to finish");
await page.waitFor(1000);
let fileName;
while (!fileName || fileName.endsWith('.crdownload')) {
await new Promise(resolve => setTimeout(resolve, 1000));
[fileName] = await util.promisify(fs.readdir)('/tmp/smartpdf');
}
const filePath = path.resolve('/tmp/smartpdf', fileName);
console.error('Downloaded file:', filePath);
console.log("download finished");
// pick up file from /tmp
console.log("reading report");
const contents = fs.readFileSync(filePath);
const stream = contents.toString('base64');
// chunk file
for (let i = 0, charsLength = stream.length; i < charsLength; i += 50000) {
await chunks.push(stream.substring(i, i + 50000));
}
const id = (new Date).getTime();
await fs.unlinkSync(filePath);
console.log("recursive call to chunk pdf...");
await chunkPDF(0, id);
// closing
console.log("hard wait");
await page.waitFor(15000);
await browser.close();
})();
smart_exporter_app\bin\smartexporter.py
cmdargs = os.path.join(phantomjs_path,"phantomjs")+ " "+f.name
i also replaced:
/opt/phantomjs/bin/phantomjs
With a symlink to:
/opt/splunk/bin/node
And I installed npm and:
npm --proxy "http://proxy:8080" i puppeteer
So this swaps out phantomjs with puppeteer which is based on chromium, perhaps I just have an old version but the rendering is pretty terrible of the dashboards.
So not working well but definitely a lot better than it was, and perhaps with a modern version of puppeteer it might work better...
Open to a better solution, but this will be good enough for now...I also tried PDF Report Capture for Splunk but could not get that working on 7.3.0 in an usable way...
Getting same error. PDF generation from dashboards works fine but schedules don't work properly.
Hi Gjanders,
if saw that it blocks in pdf generation.
does the pdf export works well if you click manually on the smart export button?
Regards,
Atef.
Yes it does but it varies depending on browser and it rarely looks exactly as it is on-screen
FYI this happens on Splunk 7.3.0 as well, this is on the default / example dashboard built into the app!