Category: Google Cloud

WordPress REST API retrieve data and email it as an xls using Google Sheets

31st July 2018 by Alan Leave a Comment

Google Script to read a WordPress REST API endpoint into a Google Sheet and then email it ( no titles in this one, see other tutorials for titles )

function onOpen() {
  // setup a custom menu
  SpreadsheetApp.getUi() 
      .createMenu('Barcodes')
      .addItem('Get Data', 'getPosts')
      .addToUi();
  setupSheet();
}

function setupSheet() {
  // just clear sheet and set &amp; style headings
 var ss = SpreadsheetApp.getActiveSpreadsheet();
  // set the tab name - just for neatness
 var sheetname = ss.getSheets()[0].setName('Data');
  // clear the sheet
 var range = ss.getDataRange();
 range.clear();
}


function getPosts(){
  setupSheet();
  // loop here as we need get back posts in chunks, Rest API max is 100 post
  offset = 0;
  per_page = 10;
  while ( (got=getPage(offset,per_page)) > 0 ) {
    offset+=got;
  }
  SpreadsheetApp.flush();
  sendGoogleSpreadsheetAsExcel();
}

function getPage(offset,per_page) {
  // gets posts in chunks of per_page
  var ui = SpreadsheetApp.getUi();  // used for error messages
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  
  var options = {
      'method': 'get',
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  var apiHost = 'https://YOURDOMAIN/wp-json';   // set to your own domain
  url = apiHost + '/wp/v2/ENDPOINT?per_page='+per_page+'&amp;offset='+offset;  // set to appropriate END POINT
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    // loop through the map and output to sheet
    for (i = 0; i &lt; data.length; i++) {
     row=offset+i+1;  
     ss.getRange('A'+row).setValue(data[i].title.rendered);   // work out the data structure by looking at the JSON
     ss.getRange('B'+row).setValue(data[i].barcode_id);       // work out the data structure by looking at the JSON
  }
    return data.length;
  } catch(error) {
    var result = ui.alert( error.toString());
  }
  return 0;
}

function sendGoogleSpreadsheetAsExcel(){
  
  try {
    var ss = SpreadsheetApp.getActiveSpreadsheet();
    var url = "https://docs.google.com/feeds/download/spreadsheets/Export?key=" + ss.getId() + "&amp;exportFormat=xlsx";
    var params = {
      method      : "get",
      headers     : {"Authorization": "Bearer " + ScriptApp.getOAuthToken()},
      muteHttpExceptions: true
    };
    var blob = UrlFetchApp.fetch(url, params).getBlob();
    blob.setName(ss.getName() + ".xlsx");
    // change the EMAIL address
    MailApp.sendEmail("youremail@example.com", "Google Sheet to Excel", "The XLSX file is attached", {attachments: [blob]});   
  } catch (f) {
    Logger.log(f.toString());
  }
}

function onOpen() {

// setup a custom menu

SpreadsheetApp.getUi()

.createMenu('Barcodes')

.addItem('Get Data', 'getPosts')

.addToUi();

setupSheet();

}

function setupSheet() {

// just clear sheet and set & style headings

var ss = SpreadsheetApp.getActiveSpreadsheet();

// set the tab name - just for neatness

var sheetname = ss.getSheets()[0].setName('Data');

// clear the sheet

var range = ss.getDataRange();

range.clear();

}

function getPosts(){

setupSheet();

// loop here as we need get back posts in chunks, Rest API max is 100 post

offset = 0;

per_page = 10;

while ( (got=getPage(offset,per_page)) > 0 ) {

offset+=got;

}

SpreadsheetApp.flush();

sendGoogleSpreadsheetAsExcel();

}

function getPage(offset,per_page) {

// gets posts in chunks of per_page

var ui = SpreadsheetApp.getUi(); // used for error messages

var ss = SpreadsheetApp.getActiveSpreadsheet();

var options = {

'method': 'get',

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = 'https://YOURDOMAIN/wp-json'; // set to your own domain

url = apiHost + '/wp/v2/ENDPOINT?per_page='+per_page+'&offset='+offset; // set to appropriate END POINT

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

// loop through the map and output to sheet

for (i = 0; i < data.length; i++) {

row=offset+i+1;

ss.getRange('A'+row).setValue(data[i].title.rendered); // work out the data structure by looking at the JSON

ss.getRange('B'+row).setValue(data[i].barcode_id); // work out the data structure by looking at the JSON

}

return data.length;

} catch(error) {

var result = ui.alert( error.toString());

}

return 0;

}

function sendGoogleSpreadsheetAsExcel(){

try {

var ss = SpreadsheetApp.getActiveSpreadsheet();

var url = "https://docs.google.com/feeds/download/spreadsheets/Export?key=" + ss.getId() + "&exportFormat=xlsx";

var params = {

method : "get",

headers : {"Authorization": "Bearer " + ScriptApp.getOAuthToken()},

muteHttpExceptions: true

};

var blob = UrlFetchApp.fetch(url, params).getBlob();

blob.setName(ss.getName() + ".xlsx");

// change the EMAIL address

MailApp.sendEmail("youremail@example.com", "Google Sheet to Excel", "The XLSX file is attached", {attachments: [blob]});

} catch (f) {

Logger.log(f.toString());

}

Putting it all together – authenticate, read and update WordPress from a spreadsheet

16th January 2018 by Alan 1 Comment

This is my fourth post in my series of how to build a script to manage WordPress data via the API and a spreadsheet. If you have arrived here without looking at the others you probably should start at the beginning here.

In this tutorial I’m going to handle updating and then pull it all together ( read / authenticate / update ) into a script that initially manages Yoast titles via the spreadsheet.

Something to bear in mind, it that Google Scripts has a maximum execution time of around 5 minutes, whilst reading data into the spreadsheet from my servers I can read thousands of posts in a minute or so, for updating, on my servers I can only get about 500 updates called in the 5 minute before time out. In this script I’m not trying to be clever on timeout / restarting as you can do that manually, but it is possible with timed triggers – but I leave that for advanced posts.

As we have already built the getPosts function in post one, and modified it for meta in post two, and done the basics for JWT authorisation in post three I will concentrate on the updating posts first .

First of all we need a way to know a cell has changed, as we don’t want to scan maybe thousands of rows, trying to update things that haven’t actually changed.

The way I tackle this, is simply to set the background colour of a changed yoast title cell to red, this gives us a nice visual clue, and later when it has been updated set the background to green. This also enables you to re run the script if it breaks ( e.g. timeout ) as the process will pick up on the remaining red cells. This is easy to set using the reserved onEdit(e) function as follow:

function onEdit(e){
  // If it is in the yoast title cell areas and changed turn it red
  var range = e.range;
  var thisCol = range.getColumn();
  var thisRow = range.getRow();
  if ( thisRow > 1 && thisCol == 3 ) {
    range.setBackground('#ff0000');
  }
}

function onEdit(e){

// If it is in the yoast title cell areas and changed turn it red

var range = e.range;

var thisCol = range.getColumn();

var thisRow = range.getRow();

if ( thisRow > 1 && thisCol == 3 ) {

range.setBackground('#ff0000');

}

Now we need a control loop for the updating of posts, this loop traverses the rows, looking for red backgrounds and calling the individual posts to update then, and set them to green

function updatePosts(){
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  var sheet = ss.getSheets()[0];
  var range = sheet.getRange(2, 1, sheet.getMaxRows()-1, 4);   // row count starts at 1 but arrays start at zero hence need to -1
  var bg = range.getBackgrounds();         // gets 2 dimensional array of background
  for (i = 0; i < bg.length; i++) {
    if ( bg[i][2] == '#ff0000') {          // array 2 is column 3 = our yoast title - is it red?
      var r2 = sheet.getRange(i+2, 1, 1, 4);   // yes so set the range for that row
      var line = r2.getValues();               // and get a two dimension array of values
      if (line[0][0] != '' ) {                 // just check we haven't got a blank post ID to stop breaking things
          updateLine(line[0]);                 // call our line updater
          var r3 = sheet.getRange(i+2, 3);     // now get the specific range for the yoast title alone
          r3.setBackground('#00ff00');         // and set it green
      }
    }
  }
}

function updatePosts(){

var ss = SpreadsheetApp.getActiveSpreadsheet();

var sheet = ss.getSheets()[0];

var range = sheet.getRange(2, 1, sheet.getMaxRows()-1, 4); // row count starts at 1 but arrays start at zero hence need to -1

var bg = range.getBackgrounds(); // gets 2 dimensional array of background

for (i = 0; i < bg.length; i++) {

if ( bg[i][2] == '#ff0000') { // array 2 is column 3 = our yoast title - is it red?

var r2 = sheet.getRange(i+2, 1, 1, 4); // yes so set the range for that row

var line = r2.getValues(); // and get a two dimension array of values

if (line[0][0] != '' ) { // just check we haven't got a blank post ID to stop breaking things

updateLine(line[0]); // call our line updater

var r3 = sheet.getRange(i+2, 3); // now get the specific range for the yoast title alone

r3.setBackground('#00ff00'); // and set it green

}

And we need the update calls for the specific lines ( rows)

function updateLine(line){
  var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json';  // get the domain that maybe was stored
  var ui = SpreadsheetApp.getUi();
  
  // see if we have a valid token
   var headers = {
    "Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token')   // JWT plugin 
    };
   var options = {
      'method': 'post',
      "contentType" : "application/json",
      "headers": headers,
      'muteHttpExceptions': false
    }
    url = apiHost + '/jwt-auth/v1/token/validate';   
    try {
      var response = UrlFetchApp.fetch(url, options);
    } catch(error) {
      showDialog();
      return;
    }
 
  var payload = {
     meta: {
      _yoast_wpseo_title: line[2]
      }
    }
  
  var options = {
      'method': 'post',
      'headers': headers,
      'payload': JSON.stringify(payload),
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  
  var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json'; 
  url = apiHost + '/wp/v2/posts/'+line[0];   // the api call
  try {
    var response = UrlFetchApp.fetch(url, options);
  } catch(error) {
    var result = ui.alert( error.toString());
  }
}

function updateLine(line){

var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json'; // get the domain that maybe was stored

var ui = SpreadsheetApp.getUi();

// see if we have a valid token

var headers = {

"Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token') // JWT plugin

};

var options = {

'method': 'post',

"contentType" : "application/json",

"headers": headers,

'muteHttpExceptions': false

}

url = apiHost + '/jwt-auth/v1/token/validate';

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

showDialog();

return;

}

var payload = {

meta: {

_yoast_wpseo_title: line[2]

}

var options = {

'method': 'post',

'headers': headers,

'payload': JSON.stringify(payload),

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json';

url = apiHost + '/wp/v2/posts/'+line[0]; // the api call

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

var result = ui.alert( error.toString());

}

And of course we need a menu item to run the updates from, this time we won’t create a separate menu item as if we want to put a script into the chrome web store as a spreadsheet add on (even for your won G Suite domain) , you can’t have a top level menu. It as to be under the add-ons menu.

function onInstall(e) {   // required for add-ons
  onOpen(e);
}
function onOpen(e) {
  // setup a custom menu
  SpreadsheetApp.getUi().createAddonMenu() 
      .addItem('Get Posts', 'getPosts')
      .addItem('Update Posts', 'updatePosts')
      .addToUi();
  setupSheet();
}

function onInstall(e) { // required for add-ons

onOpen(e);

}

function onOpen(e) {

// setup a custom menu

SpreadsheetApp.getUi().createAddonMenu()

.addItem('Get Posts', 'getPosts')

.addItem('Update Posts', 'updatePosts')

.addToUi();

setupSheet();

}

So bringing the elements of the other post in we end up with

function onInstall(e) {   // required for add-ons
  onOpen(e);
}
function onOpen(e) {
  // setup a custom menu
  SpreadsheetApp.getUi().createAddonMenu() 
      .addItem('Get Posts', 'getPosts')
      .addItem('Update Posts', 'updatePosts')
      .addToUi();
  setupSheet();
}

function onEdit(e){
  // If it is in the yoast title cell areas and changed turn it red
  var range = e.range;
  var thisCol = range.getColumn();
  var thisRow = range.getRow();
  if ( thisRow > 1 && thisCol == 3 ) {
    range.setBackground('#ff0000');
  }
}

function setupSheet() {
  // just clear sheet and set & style headings
 var ss = SpreadsheetApp.getActiveSpreadsheet();
  // set the tab to post - just for neatness
 var sheetname = ss.getSheets()[0].setName('Posts');
  // clear the sheet
 var range = ss.getDataRange();
 range.clear();
 //set headings
  ss.getRange('A1').setValue('Post ID');
  ss.getRange('B1').setValue('Title');
  ss.getRange('C1').setValue('Yoast Title');
  ss.getRange('D1').setValue('Updated');
  
  ss.getRange('A1:C1').setFontWeight('bold');
  ss.getRange('A1:C1').setFontStyle('italic');
 

}

function showDialog() {
  var html = HtmlService.createHtmlOutputFromFile('loginpage')   // dont forget we need an html form in the project
      .setWidth(200)
      .setHeight(200);
  SpreadsheetApp.getUi().showModalDialog(html, 'Connect to WordPress');
}
 
function processLogin(myForm) {
  var payload = {
    username: myForm.un,
    password: myForm.pw
  }
  var options = {
      'method': 'post',
      "contentType" : "application/json",
      'payload': JSON.stringify(payload),
      'muteHttpExceptions': false
    }
  PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain);
  var apiHost = myForm.domain+'/wp-json';
  var ui = SpreadsheetApp.getUi(); // Same variations.
  url = apiHost + '/jwt-auth/v1/token';
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    PropertiesService.getDocumentProperties().setProperty('token', data.token);
    var result = ui.alert( 'got valid connection token' );
  } catch(error) {
    var result = ui.alert( 'invalid user or password: url: '+ url +' err: ' + error.toString());
  }
  updatePosts();
}

function getPosts(){
  setupSheet();
  // loop here as we need get back posts in chunks, Rest API max is 100 post
  offset = 0;
  per_page = 10;
  while ( (got=getPage(offset,per_page)) > 0 ) {
    offset+=got;
  }
}


function getPage(offset,per_page) {
  // gets posts in chunks of per_page
  var ui = SpreadsheetApp.getUi();  // used for error messages
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  
  var options = {
      'method': 'get',
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  var apiHost = 'http://mytestdomain/wp-json';   // set to your own domain
  url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    // loop through the map and output to sheet
    // performance can be improved by storing this in an array and returning it to getPosts to display in one go rather than line by line
    for (i = 0; i < data.length; i++) {
     row=offset+i+2;  // set the row to make sure it is below header and takes into account the paging
     ss.getRange('A'+row).setValue(data[i].id);
     ss.getRange('B'+row).setValue(data[i].title.rendered);
     ss.getRange('C'+row).setValue(data[i].meta._yoast_wpseo_title);
     ss.getRange('D'+row).setValue(data[i].modified);
  }
    return data.length;
  } catch(error) {
    var result = ui.alert( error.toString());
  }
  return 0;
}

function updatePosts(){
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  var sheet = ss.getSheets()[0];
  var range = sheet.getRange(2, 1, sheet.getMaxRows()-1, 4);   // row count starts at 1 but arrays start at zero hence need to -1
  var bg = range.getBackgrounds();         // gets 2 dimensional array of background
  for (i = 0; i < bg.length; i++) {
    if ( bg[i][2] == '#ff0000') {          // array 2 is column 3 = our yoast title - is it red?
      var r2 = sheet.getRange(i+2, 1, 1, 4);   // yes so set the range for that row
      var line = r2.getValues();               // and get a two dimension array of values
      if (line[0][0] != '' ) {                 // just check we haven't got a blank post ID to stop breaking things
          updateLine(line[0]);                 // call our line updater
          var r3 = sheet.getRange(i+2, 3);     // now get the specific range for the yoast title alone
          r3.setBackground('#00ff00');         // and set it green
      }
    }
  }
}

function updateLine(line){
  var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json';  // get the domain that maybe was stored
  var ui = SpreadsheetApp.getUi();
  
  // see if we have a valid token
   var headers = {
    "Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token')   // JWT plugin 
    };
   var options = {
      'method': 'post',
      "contentType" : "application/json",
      "headers": headers,
      'muteHttpExceptions': false
    }
    url = apiHost + '/jwt-auth/v1/token/validate';   
    try {
      var response = UrlFetchApp.fetch(url, options);
    } catch(error) {
      showDialog();
      return;
    }
 
  var payload = {
     meta: {
      _yoast_wpseo_title: line[2]
      }
    }
  
  var options = {
      'method': 'post',
      'headers': headers,
      'payload': JSON.stringify(payload),
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  
  var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json'; 
  url = apiHost + '/wp/v2/posts/'+line[0];   // the api call
  try {
    var response = UrlFetchApp.fetch(url, options);
  } catch(error) {
    var result = ui.alert( error.toString());
  }
}

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

function onInstall(e) { // required for add-ons

onOpen(e);

}

function onOpen(e) {

// setup a custom menu

SpreadsheetApp.getUi().createAddonMenu()

.addItem('Get Posts', 'getPosts')

.addItem('Update Posts', 'updatePosts')

.addToUi();

setupSheet();

}

function onEdit(e){

// If it is in the yoast title cell areas and changed turn it red

var range = e.range;

var thisCol = range.getColumn();

var thisRow = range.getRow();

if ( thisRow > 1 && thisCol == 3 ) {

range.setBackground('#ff0000');

}

function setupSheet() {

// just clear sheet and set & style headings

var ss = SpreadsheetApp.getActiveSpreadsheet();

// set the tab to post - just for neatness

var sheetname = ss.getSheets()[0].setName('Posts');

// clear the sheet

var range = ss.getDataRange();

range.clear();

//set headings

ss.getRange('A1').setValue('Post ID');

ss.getRange('B1').setValue('Title');

ss.getRange('C1').setValue('Yoast Title');

ss.getRange('D1').setValue('Updated');

ss.getRange('A1:C1').setFontWeight('bold');

ss.getRange('A1:C1').setFontStyle('italic');

}

function showDialog() {

var html = HtmlService.createHtmlOutputFromFile('loginpage') // dont forget we need an html form in the project

.setWidth(200)

.setHeight(200);

SpreadsheetApp.getUi().showModalDialog(html, 'Connect to WordPress');

}

function processLogin(myForm) {

var payload = {

username: myForm.un,

password: myForm.pw

}

var options = {

'method': 'post',

"contentType" : "application/json",

'payload': JSON.stringify(payload),

'muteHttpExceptions': false

}

PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain);

var apiHost = myForm.domain+'/wp-json';

var ui = SpreadsheetApp.getUi(); // Same variations.

url = apiHost + '/jwt-auth/v1/token';

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

PropertiesService.getDocumentProperties().setProperty('token', data.token);

var result = ui.alert( 'got valid connection token' );

} catch(error) {

var result = ui.alert( 'invalid user or password: url: '+ url +' err: ' + error.toString());

}

updatePosts();

}

function getPosts(){

setupSheet();

// loop here as we need get back posts in chunks, Rest API max is 100 post

offset = 0;

per_page = 10;

while ( (got=getPage(offset,per_page)) > 0 ) {

offset+=got;

}

function getPage(offset,per_page) {

// gets posts in chunks of per_page

var ui = SpreadsheetApp.getUi(); // used for error messages

var ss = SpreadsheetApp.getActiveSpreadsheet();

var options = {

'method': 'get',

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = 'http://mytestdomain/wp-json'; // set to your own domain

url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

// loop through the map and output to sheet

// performance can be improved by storing this in an array and returning it to getPosts to display in one go rather than line by line

for (i = 0; i < data.length; i++) {

row=offset+i+2; // set the row to make sure it is below header and takes into account the paging

ss.getRange('A'+row).setValue(data[i].id);

ss.getRange('B'+row).setValue(data[i].title.rendered);

ss.getRange('C'+row).setValue(data[i].meta._yoast_wpseo_title);

ss.getRange('D'+row).setValue(data[i].modified);

}

return data.length;

} catch(error) {

var result = ui.alert( error.toString());

}

return 0;

}

function updatePosts(){

var ss = SpreadsheetApp.getActiveSpreadsheet();

var sheet = ss.getSheets()[0];

var range = sheet.getRange(2, 1, sheet.getMaxRows()-1, 4); // row count starts at 1 but arrays start at zero hence need to -1

var bg = range.getBackgrounds(); // gets 2 dimensional array of background

for (i = 0; i < bg.length; i++) {

if ( bg[i][2] == '#ff0000') { // array 2 is column 3 = our yoast title - is it red?

var r2 = sheet.getRange(i+2, 1, 1, 4); // yes so set the range for that row

var line = r2.getValues(); // and get a two dimension array of values

if (line[0][0] != '' ) { // just check we haven't got a blank post ID to stop breaking things

updateLine(line[0]); // call our line updater

var r3 = sheet.getRange(i+2, 3); // now get the specific range for the yoast title alone

r3.setBackground('#00ff00'); // and set it green

}

function updateLine(line){

var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json'; // get the domain that maybe was stored

var ui = SpreadsheetApp.getUi();

// see if we have a valid token

var headers = {

"Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token') // JWT plugin

};

var options = {

'method': 'post',

"contentType" : "application/json",

"headers": headers,

'muteHttpExceptions': false

}

url = apiHost + '/jwt-auth/v1/token/validate';

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

showDialog();

return;

}

var payload = {

meta: {

_yoast_wpseo_title: line[2]

}

var options = {

'method': 'post',

'headers': headers,

'payload': JSON.stringify(payload),

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json';

url = apiHost + '/wp/v2/posts/'+line[0]; // the api call

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

var result = ui.alert( error.toString());

}

And don’t forget we need the login form html too

<form>
Domain inc http/https:<br>     
<input type="text" name="domain"><br>
User:<br>     
<input type="text" name="un"><br>
Password:<br> 
<input type="password" name="pw"><br>
<input type="button" value="connect" onclick="google.script.run.processLogin(this.form)" />
</form>

<form>

Domain inc http/https:<br>

User:<br>

Password:<br>

</form>

So if you open a spreadsheet, go to tools > script editor and put the above code in you should have a bulk Yoast SEO title editor.

Don’t forget the security gotchas, like WordFence and Mod Security mentioned in post three, and don’t forget to create the custom code plugin to show meta mentioned in post 2.

This script could easily be extended to handle meta description etc. and the performance tweaks done, to make it a handy SEO tool.

Your thoughts and comments welcome.

I have already built a script for WooCommerce to update regular prices in bulk, it uses many of the same principals here. Let me know if you are interested and I will write about it.

How to authenticate WordPress Rest API from Google Sheets / Scripts

15th January 2018 by Alan 2 Comments

This is the third tutorial in my series relating to integrating Google Sheets and WordPress using the REST API.

The first post covered reading basic post information into a spreadsheet. The second post was about getting meta data from the REST API, which can be a little tricky.

This one covers authentication. Basically as an external app you need have the right credentials before you can carry out update type of calls. So in preparation for the fourth post were we move on to update data, we will explore one way of authenticating.

The developer handbook covers cookie authentication in some detail, but as we are not within the WordPress world here we have to go down the page and see the throw away comment that there are plugins that can help, and mentions OAuth 1.0a Server, Application Passwords, and JSON Web Tokens.

For this tutorial we will work with JSON Web Tokens.

First you need to install and set up the JSON Web Tokens plugin.

To set up the plugin, you will need to edit your wp-config.php and also possibly your .htaccess file. To get easy editing access I suggest you install the ICEcoder plugin follow this link for details. Read the plugin instructions to set these up correctly.

The basic flow of JSON Webtokens is to pass your username & password once and then getting a token back that authenticates the subsequent calls.

The first thing we need to do is create an html file that will be displayed as a login modal to get the user / password.

In you script editor ( if you are lost refer to and work though my first post ) go to File > New > HTML file and in the pop up type login page, you will now have an html temple – delete the html and replace with the following

<form>
Domain inc http/https:<br>     
<input type="text" name="domain"><br>
User:<br>     
<input type="text" name="un"><br>
Password:<br> 
<input type="password" name="pw"><br>
<input type="button" value="connect" onclick="google.script.run.processLogin(this.form)" />
</form>

<form>

Domain inc http/https:<br>

User:<br>

Password:<br>

</form>

This is a straight forward html form, with an onclick action to call a script function processLogin. I haven’t spent any error styling it, feel free.

The next is a working script that includes authentication. Please be warned, if you run this as it is it is hard coded to change the title of post ID one – be aware, and tweak it to your appropriate test post. In the next post I will rework the script to take updates from the spreadsheet, read the following code, and I will explain it further down.

function onOpen() {
  SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
      .createMenu('WordPress')
      .addItem('Test Update', 'testUpdate')
      .addToUi();
}

function showDialog() {
  var html = HtmlService.createHtmlOutputFromFile('loginpage')
      .setWidth(200)
      .setHeight(200);
  SpreadsheetApp.getUi().showModalDialog(html, 'Connect to WordPress');
}

function processLogin(myForm) {
var payload = {
    username: myForm.un,
    password: myForm.pw
  }
var options = {
      'method': 'post',
      "contentType" : "application/json",
      'payload': JSON.stringify(payload),
      'muteHttpExceptions': false
    }
PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain);
var apiHost = myForm.domain+'/wp-json';
var ui = SpreadsheetApp.getUi(); // Same variations.
url = apiHost + '/jwt-auth/v1/token';
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    PropertiesService.getDocumentProperties().setProperty('token', data.token);
    var result = ui.alert( 'got valid connection token' );
  } catch(error) {
    var result = ui.alert( 'invalid user or password: url: '+ url +' err: ' + error.toString());
  }
  testUpdate();
}

function testUpdate() {
  var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json';
  var ui = SpreadsheetApp.getUi();
  
  // see if we have a valid token
   var headers = {
    "Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token')
    };
   var options = {
      'method': 'post',
      "contentType" : "application/json",
      "headers": headers,
      'muteHttpExceptions': false
    }
    url = apiHost + '/jwt-auth/v1/token/validate';   
    try {
      var response = UrlFetchApp.fetch(url, options);
    } catch(error) {
      showDialog();
      return;
    }
  
  // if we got here we have a valid token
    var payload = {
      title: 'Test Title',     // ** warning **  this may change your data
    }
    var options = {
      'method': 'post',
      "contentType" : "application/json",
      "headers": headers,
      'payload': JSON.stringify(payload),
      'muteHttpExceptions': false
    }
    
    url = apiHost + '/wp/v2/posts/1';    // ** warning **  hard code your test post id here instead of 1
    try {
      var response = UrlFetchApp.fetch(url, options);
    } catch(error) {
      var result = ui.alert( error.toString());
    }
}

function onOpen() {

SpreadsheetApp.getUi() // Or DocumentApp or FormApp.

.createMenu('WordPress')

.addItem('Test Update', 'testUpdate')

.addToUi();

}

function showDialog() {

var html = HtmlService.createHtmlOutputFromFile('loginpage')

.setWidth(200)

.setHeight(200);

SpreadsheetApp.getUi().showModalDialog(html, 'Connect to WordPress');

}

function processLogin(myForm) {

var payload = {

username: myForm.un,

password: myForm.pw

}

var options = {

'method': 'post',

"contentType" : "application/json",

'payload': JSON.stringify(payload),

'muteHttpExceptions': false

}

PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain);

var apiHost = myForm.domain+'/wp-json';

var ui = SpreadsheetApp.getUi(); // Same variations.

url = apiHost + '/jwt-auth/v1/token';

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

PropertiesService.getDocumentProperties().setProperty('token', data.token);

var result = ui.alert( 'got valid connection token' );

} catch(error) {

var result = ui.alert( 'invalid user or password: url: '+ url +' err: ' + error.toString());

}

testUpdate();

}

function testUpdate() {

var apiHost = PropertiesService.getDocumentProperties().getProperty('domain')+'/wp-json';

var ui = SpreadsheetApp.getUi();

// see if we have a valid token

var headers = {

"Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token')

};

var options = {

'method': 'post',

"contentType" : "application/json",

"headers": headers,

'muteHttpExceptions': false

}

url = apiHost + '/jwt-auth/v1/token/validate';

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

showDialog();

return;

}

// if we got here we have a valid token

var payload = {

title: 'Test Title', // ** warning ** this may change your data

}

var options = {

'method': 'post',

"contentType" : "application/json",

"headers": headers,

'payload': JSON.stringify(payload),

'muteHttpExceptions': false

}

url = apiHost + '/wp/v2/posts/1'; // ** warning ** hard code your test post id here instead of 1

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

var result = ui.alert( error.toString());

}

Before I explain the details of the above script there are some security gotchas – worth mentioning.

Mod Security

Many hosts, rightly run mod_security on apache to trap hacking exploits. Unfortunately some implementations of mod_security come up with the error JSON support was not enabled if you are getting 403 errors when trying to get the token of apply an update check your e=server error logs. Fixing this issue is very technical and down to your host.

WordFence

WordFence is very aggressive in stoping JSON payloads too, and if you get 403 errors and you have WordFence turned on, try deactivating it. If you have WordFence premium, their support may be able to help you, or maybe you will have to try different security software.

Other Security Software

I haven’t tested all security software, so can’t say, but if you are getting 403 or 404 or indeed any connection issues, try turning off your security software. Please let me know in the comments what you find.

function onOpen() {
  SpreadsheetApp.getUi() // Or DocumentApp or FormApp.
      .createMenu('WordPress')
      .addItem('Test Update', 'testUpdate')
      .addToUi();
}

function onOpen() {

SpreadsheetApp.getUi() // Or DocumentApp or FormApp.

.createMenu('WordPress')

.addItem('Test Update', 'testUpdate')

.addToUi();

}

This creates a new menu on the spreadsheet and allows you to call the test function.

function showDialog() {
  var html = HtmlService.createHtmlOutputFromFile('loginpage')
      .setWidth(200)
      .setHeight(200);
  SpreadsheetApp.getUi().showModalDialog(html, 'Connect to WordPress');
}

function showDialog() {

var html = HtmlService.createHtmlOutputFromFile('loginpage')

.setWidth(200)

.setHeight(200);

SpreadsheetApp.getUi().showModalDialog(html, 'Connect to WordPress');

}

This function displays the html page we set up in a modal, we call this when we have an invalid token. Remember the html form calls the processLogin function on submit.

function processLogin(myForm) {
var payload = {
    username: myForm.un,
    password: myForm.pw
  }
var options = {
      'method': 'post',
      "contentType" : "application/json",
      'payload': JSON.stringify(payload),
      'muteHttpExceptions': false
    }
PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain);
var apiHost = myForm.domain+'/wp-json';
var ui = SpreadsheetApp.getUi(); // Same variations.
url = apiHost + '/jwt-auth/v1/token';
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    PropertiesService.getDocumentProperties().setProperty('token', data.token);
    var result = ui.alert( 'got valid connection token' );
  } catch(error) {
    var result = ui.alert( 'invalid user or password: url: '+ url +' err: ' + error.toString());
  }
  testUpdate();
}

function processLogin(myForm) {

var payload = {

username: myForm.un,

password: myForm.pw

}

var options = {

'method': 'post',

"contentType" : "application/json",

'payload': JSON.stringify(payload),

'muteHttpExceptions': false

}

PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain);

var apiHost = myForm.domain+'/wp-json';

var ui = SpreadsheetApp.getUi(); // Same variations.

url = apiHost + '/jwt-auth/v1/token';

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

PropertiesService.getDocumentProperties().setProperty('token', data.token);

var result = ui.alert( 'got valid connection token' );

} catch(error) {

var result = ui.alert( 'invalid user or password: url: '+ url +' err: ' + error.toString());

}

testUpdate();

}

The key items here are PropertiesService.getDocumentProperties().setProperty('domain', myForm.domain); where we stoore the domain in persistent storage.

If we successfully get a token when calling the token endpoint, we also store that PropertiesService.getDocumentProperties().setProperty('token', data.token);

Finally, if we succeed or not we recall testUpdate(); as we want to drop back into that process, effectively looping until we get a valid token. See the next part

function testUpdate() {
  var apiHost = PropertiesService.getDocumenttProperties().getProperty('domain')+'/wp-json';
  var ui = SpreadsheetApp.getUi();
  
  // see if we have a valid token
   var headers = {
    "Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token')
    };
   var options = {
      'method': 'post',
      "contentType" : "application/json",
      "headers": headers,
      'muteHttpExceptions': false
    }
    url = apiHost + '/jwt-auth/v1/token/validate';   
    try {
      var response = UrlFetchApp.fetch(url, options);
    } catch(error) {
      showDialog();
      return;
    }
  
  // if we got here we have a valid token
    var payload = {
      title: 'Test Title',     // ** warning **  this may change your data
    }
    var options = {
      'method': 'post',
      "contentType" : "application/json",
      "headers": headers,
      'payload': JSON.stringify(payload),
      'muteHttpExceptions': false
    }
    
    url = apiHost + '/wp/v2/posts/1';    // ** warning **  hard code your test post id here instead of 1
    try {
      var response = UrlFetchApp.fetch(url, options);
    } catch(error) {
      var result = ui.alert( error.toString());
    }
}

function testUpdate() {

var apiHost = PropertiesService.getDocumenttProperties().getProperty('domain')+'/wp-json';

var ui = SpreadsheetApp.getUi();

// see if we have a valid token

var headers = {

"Authorization" : "Bearer " + PropertiesService.getDocumentProperties().getProperty('token')

};

var options = {

'method': 'post',

"contentType" : "application/json",

"headers": headers,

'muteHttpExceptions': false

}

url = apiHost + '/jwt-auth/v1/token/validate';

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

showDialog();

return;

}

// if we got here we have a valid token

var payload = {

title: 'Test Title', // ** warning ** this may change your data

}

var options = {

'method': 'post',

"contentType" : "application/json",

"headers": headers,

'payload': JSON.stringify(payload),

'muteHttpExceptions': false

}

url = apiHost + '/wp/v2/posts/1'; // ** warning ** hard code your test post id here instead of 1

try {

var response = UrlFetchApp.fetch(url, options);

} catch(error) {

var result = ui.alert( error.toString());

}

This is two calls, first a call to validate the token, if that fails, that is where we call the dialogue.

If we have a valid token, we move on to a test update. See how the payload is made up for update.

So, now we have a structure that works for authentication, prompting for domain, user & password if we haven’t got a tken, and applying an update if we have.

In the next post, I will ties together getting posts, including Yoast meta title, authenticating and applying updates from changes made on the spreadsheet.

I hope you are finding these useful. Comments appreciated.

Getting post meta via WordPress REST API

12th January 2018 by Alan 9 Comments

This the second post post in my series about getting and updating WordPress data via a spreadsheet. The first post focused on the basics of Google Apps Scripting and the API call to the WP REST API – if you haven’t seen it you can read it here.

This post is about meta data. In case you are not really clear, meta data in WordPress post terms are any extra data that isn’t part of the WordPress post schema. More often or not this data is created and maintained by plugins.

To give you a clear example, if say you use Yoast SEO plugin, the Yoast SEO version of the post title is stored in meta data item called _yoast_wpseo_title these are also known as custom fields, and WordPress treats custom fields starting with an _ as hidden.

I read many posts on howto access meta and most of them were wrong and out dated. The WP Rest documentation tells you about problems with meta and also warns you they are changing their thoughts on this – so beware – this may even be out of date soon.

If you look at the ‘posts schema’ you will see an entry for meta, but more often than not, you will see an entry "meta":[] i.e. empty, even if it is not e.g. curl http://demo.wp-api.org/wp-json/wp/v2/posts/1

For unsetialized ( i.e. text string meta) the solution is much simpler that many blog posts I read make out, you just need to tell WordPress that the meta can be exposed in the API.

To do this, you are going to need to write ( or copy at least ) some PHP code. Some tutorials, will tell you that you add extra code to your theme’s functions.php, whilst this will work, this is not the right way to do things, themes are for layouts, plugins are for functions. If you add it to your (child) theme and one day you change your theme then the extra features will break. So write a plugin, its not hard.

Before you write the plugin, you will have been testing on a website on the internet ( you have to as Google can’t see your local host ), so here is a quick and easy way to get editing on your remote site without faffing around with FTP etc. Install ICEcoder with this plugin to be able to follow along.

Once you are in the editor navigate to the wp-content/plugins folder, right click on plugins and select ‘New File’ and give your plugin a name like myfunctions.php .

The copy the code below and paste it into the code area and ctrl-s to save it.

<?php
/**
 * Plugin Name:       My Custom Code
 * Plugin URI:        http://fullworks.net/wordpress-plugins
 * Description:       Custom code
 * Version:           1.0
 * Author:            Fullworks
 * Author URI:        http://fullworks.net/
 *
 */
if ( ! defined( 'WPINC' ) ) {
	die;
}
add_action( 'rest_api_init', 'register_posts_meta_field' );

function register_posts_meta_field() {
    $object_type = 'post';
    $args1 = array( // Validate and sanitize the meta value.
        // Note: currently (4.7) one of 'string', 'boolean', 'integer',
        // 'number' must be used as 'type'. The default is 'string'.
        'type'         => 'string',
        // Shown in the schema for the meta key.
        'description'  => 'A meta key associated with a string meta value.',
        // Return a single value of the type.
        'single'       => true,
        // Show in the WP REST API response. Default: false.
        'show_in_rest' => true,
    );
    register_meta( $object_type, '_yoast_wpseo_title', $args1 );
}

<?php

/**

* Plugin Name: My Custom Code

* Plugin URI: http://fullworks.net/wordpress-plugins

* Description: Custom code

* Version: 1.0

* Author: Fullworks

* Author URI: http://fullworks.net/

if ( ! defined( 'WPINC' ) ) {

die;

}

add_action( 'rest_api_init', 'register_posts_meta_field' );

function register_posts_meta_field() {

$object_type = 'post';

$args1 = array( // Validate and sanitize the meta value.

// Note: currently (4.7) one of 'string', 'boolean', 'integer',

// 'number' must be used as 'type'. The default is 'string'.

'type' => 'string',

// Shown in the schema for the meta key.

'description' => 'A meta key associated with a string meta value.',

// Return a single value of the type.

'single' => true,

// Show in the WP REST API response. Default: false.

'show_in_rest' => true,

);

register_meta( $object_type, '_yoast_wpseo_title', $args1 );

}

Lets just take a second to explain what we are doing.

when the Rest API is initiated 'rest_api_init' we are calling our function which is basically registering _yoast_wpseo_title but this time it is setting it to show in the Rest API 'show_in_rest' => true,

OK, so when you have saved your plugin, go to your admin > plugins and find it, and activate it, hopefully it activates without errors.

So you now are ready to go, let just extend the script we wrote in the first article to also read the meta items.

If you are following this along, you will obviously need to install the Yoast SEO plugin into your remote test WordPress, and also set a few Yoast titles up in posts.

The script code would now look like this

function onOpen() {
  // setup a custom menu
  SpreadsheetApp.getUi() 
      .createMenu('WordPress')
      .addItem('Get Posts', 'getPosts')
      .addToUi();
  setupSheet();
}

function setupSheet() {
  // just clear sheet and set & style headings
 var ss = SpreadsheetApp.getActiveSpreadsheet();
  // set the tab to post - just for neatness
 var sheetname = ss.getSheets()[0].setName('Posts');
  // clear the sheet
 var range = ss.getDataRange();
 range.clear();
 //set headings
  ss.getRange('A1').setValue('Post ID');
  ss.getRange('B1').setValue('Title');
  ss.getRange('C1').setValue('Yoast Title');
  ss.getRange('D1').setValue('Updated');
  
  ss.getRange('A1:C1').setFontWeight('bold');
  ss.getRange('A1:C1').setFontStyle('italic');

}


function getPosts(){
  setupSheet();
  // loop here as we need get back posts in chunks, Rest API max is 100 post
  offset = 0;
  per_page = 10;
  while ( (got=getPage(offset,per_page)) > 0 ) {
    offset+=got;
    Logger.log('offset %s', offset);
  }
}


function getPage(offset,per_page) {
  // gets posts in chunks of per_page
  var ui = SpreadsheetApp.getUi();  // used for error messages
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  
  var options = {
      'method': 'get',
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  var apiHost = 'http://mytestdomain/wp-json';   // set to your own domain
  url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    // loop through the map and output to sheet
    for (i = 0; i < data.length; i++) {
     row=offset+i+2;  // set the row to make sure it is below header and takes into account the paging
     ss.getRange('A'+row).setValue(data[i].id);
     ss.getRange('B'+row).setValue(data[i].title.rendered);
     ss.getRange('C'+row).setValue(data[i].meta._yoast_wpseo_title);
     ss.getRange('D'+row).setValue(data[i].modified);
  }
    return data.length;
  } catch(error) {
    var result = ui.alert( error.toString());
  }
  return 0;
}

function onOpen() {

// setup a custom menu

SpreadsheetApp.getUi()

.createMenu('WordPress')

.addItem('Get Posts', 'getPosts')

.addToUi();

setupSheet();

}

function setupSheet() {

// just clear sheet and set & style headings

var ss = SpreadsheetApp.getActiveSpreadsheet();

// set the tab to post - just for neatness

var sheetname = ss.getSheets()[0].setName('Posts');

// clear the sheet

var range = ss.getDataRange();

range.clear();

//set headings

ss.getRange('A1').setValue('Post ID');

ss.getRange('B1').setValue('Title');

ss.getRange('C1').setValue('Yoast Title');

ss.getRange('D1').setValue('Updated');

ss.getRange('A1:C1').setFontWeight('bold');

ss.getRange('A1:C1').setFontStyle('italic');

}

function getPosts(){

setupSheet();

// loop here as we need get back posts in chunks, Rest API max is 100 post

offset = 0;

per_page = 10;

while ( (got=getPage(offset,per_page)) > 0 ) {

offset+=got;

Logger.log('offset %s', offset);

}

function getPage(offset,per_page) {

// gets posts in chunks of per_page

var ui = SpreadsheetApp.getUi(); // used for error messages

var ss = SpreadsheetApp.getActiveSpreadsheet();

var options = {

'method': 'get',

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = 'http://mytestdomain/wp-json'; // set to your own domain

url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

// loop through the map and output to sheet

for (i = 0; i < data.length; i++) {

row=offset+i+2; // set the row to make sure it is below header and takes into account the paging

ss.getRange('A'+row).setValue(data[i].id);

ss.getRange('B'+row).setValue(data[i].title.rendered);

ss.getRange('C'+row).setValue(data[i].meta._yoast_wpseo_title);

ss.getRange('D'+row).setValue(data[i].modified);

}

return data.length;

} catch(error) {

var result = ui.alert( error.toString());

}

return 0;

}

See what I did there, just inserted column C as a title and put .meta._yoast_wpseo_title into column C

Change the domain to your actual internet available test domain. Save and run from the spreadsheet. Refer back to the first article if you need to see how.

If you have any questions or comments, please leave them.

In the next post I will work through a method to get authentication so you can later update.

Linking WordPress to a Spreadsheet using WP REST API and Google Sheets scripts

11th January 2018 by Alan 6 Comments

This post shows you how you can link a Google Spreadsheet to a WordPress website to extract data. I will write some further posts covering authentication and updating too. However a little warning, this is a programming task and a basic understanding of javascript is really required to follow what is going on.

WordPress is now accessible to other applications through the WP REST API. Google Sheets is a free spreadsheet solution, that is very similar to Microsoft Excel, but runs in the Cloud. Being a Google App, Google Sheets has access to Google Apps Scripts, Google Apps Script is a JavaScript cloud scripting language that provides easy ways to automate tasks across Google products and third party services and build web applications. So now you can see the foundations of linking a WordPress website to a spreadsheet.

You are most likely here because you are a developer and you want to find out how to achieve getting WordPress data into a spreadsheet, however on the off-chance you are a business and you want some custom work or consultancy relating to WordPress please contact me via this link.

(If you are a developer and you have improvements or comments, please use the comment form at the bottom )

O.K. lets get started.

The first thing to do is to create a Google Sheet that is going to be the basis of your project. -> https://docs.google.com/spreadsheets

Start a new blank spreadsheet and change the title from ‘untitled spreadsheet’ to something, anything, you like.

Then, go to the menu Tools > Script Editor and open up a new script, also change that from ‘untitled project ‘ to a name you like.

Next simply replace

function myFunction() {
  
}

function myFunction() {

}

with the following and File > Save All ( I will explain the code later down )

function onOpen() {
  // setup a custom menu
  SpreadsheetApp.getUi() 
      .createMenu('WordPress')
      .addItem('Get Posts', 'getPosts')
      .addToUi();
  setupSheet();
}

function setupSheet() {
  // just clear sheet and set & style headings
 var ss = SpreadsheetApp.getActiveSpreadsheet();
  // set the tab to post - just for neatness
 var sheetname = ss.getSheets()[0].setName('Posts');
  // clear the sheet
 var range = ss.getDataRange();
 range.clear();
 //set headings
  ss.getRange('A1').setValue('Post ID');
  ss.getRange('B1').setValue('Title');
  ss.getRange('C1').setValue('Updated');
  
  ss.getRange('A1:C1').setFontWeight('bold');
  ss.getRange('A1:C1').setFontStyle('italic');

}


function getPosts(){
  setupSheet();
  // loop here as we need get back posts in chunks, Rest API max is 100 post
  offset = 0;
  per_page = 10;
  while ( (got=getPage(offset,per_page)) > 0 ) {
    offset+=got;
    Logger.log('offset %s', offset);
  }
}


function getPage(offset,per_page) {
  // gets posts in chunks of per_page
  var ui = SpreadsheetApp.getUi();  // used for error messages
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  
  var options = {
      'method': 'get',
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  var apiHost = 'http://demo.wp-api.org/wp-json';   // set to your own domain
  url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    // loop through the map and output to sheet
    for (i = 0; i < data.length; i++) {
     row=offset+i+2;  // set the row to make sure it is below header and takes into account the paging
     ss.getRange('A'+row).setValue(data[i].id);
     ss.getRange('B'+row).setValue(data[i].title.rendered);
     ss.getRange('C'+row).setValue(data[i].modified);
  }
    return data.length;
  } catch(error) {
    var result = ui.alert( error.toString());
  }
  return 0;
}

function onOpen() {

// setup a custom menu

SpreadsheetApp.getUi()

.createMenu('WordPress')

.addItem('Get Posts', 'getPosts')

.addToUi();

setupSheet();

}

function setupSheet() {

// just clear sheet and set & style headings

var ss = SpreadsheetApp.getActiveSpreadsheet();

// set the tab to post - just for neatness

var sheetname = ss.getSheets()[0].setName('Posts');

// clear the sheet

var range = ss.getDataRange();

range.clear();

//set headings

ss.getRange('A1').setValue('Post ID');

ss.getRange('B1').setValue('Title');

ss.getRange('C1').setValue('Updated');

ss.getRange('A1:C1').setFontWeight('bold');

ss.getRange('A1:C1').setFontStyle('italic');

}

function getPosts(){

setupSheet();

// loop here as we need get back posts in chunks, Rest API max is 100 post

offset = 0;

per_page = 10;

while ( (got=getPage(offset,per_page)) > 0 ) {

offset+=got;

Logger.log('offset %s', offset);

}

function getPage(offset,per_page) {

// gets posts in chunks of per_page

var ui = SpreadsheetApp.getUi(); // used for error messages

var ss = SpreadsheetApp.getActiveSpreadsheet();

var options = {

'method': 'get',

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = 'http://demo.wp-api.org/wp-json'; // set to your own domain

url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

// loop through the map and output to sheet

for (i = 0; i < data.length; i++) {

row=offset+i+2; // set the row to make sure it is below header and takes into account the paging

ss.getRange('A'+row).setValue(data[i].id);

ss.getRange('B'+row).setValue(data[i].title.rendered);

ss.getRange('C'+row).setValue(data[i].modified);

}

return data.length;

} catch(error) {

var result = ui.alert( error.toString());

}

return 0;

}

Once you have saved the script, return to your spreadsheet tab and refresh it. You will now see an extra menu tab – WordPress

Under the WordPress tab is a sub menu – Get Posts – click on it and see the magic happen.

Lets break down the code and explain

function onOpen() {
  // setup a custom menu
  SpreadsheetApp.getUi() 
      .createMenu('WordPress')
      .addItem('Get Posts', 'getPosts')
      .addToUi();
  setupSheet();
}

function onOpen() {

// setup a custom menu

SpreadsheetApp.getUi()

.createMenu('WordPress')

.addItem('Get Posts', 'getPosts')

.addToUi();

setupSheet();

}

This function is fired automatically when the spreadsheet page is opened ( or refreshed ). It builds the menu ‘WordPress’ and a sub menu ‘Get Posts’ that calls our processing function getPosts().

I also have a function setupSheet() called which I will explain next.

function setupSheet() {
  // just clear sheet and set & style headings
 var ss = SpreadsheetApp.getActiveSpreadsheet();
  // set the tab to post - just for neatness
 var sheetname = ss.getSheets()[0]setName('Posts');
  // clear the sheet
 var range = ss.getDataRange();
 range.clear();
 //set headings
  ss.getRange('A1').setValue('Post ID');
  ss.getRange('B1').setValue('Title');
  ss.getRange('C1').setValue('Updated');
  
  ss.getRange('A1:C1').setFontWeight('bold');
  ss.getRange('A1:C1').setFontStyle('italic');

}

function setupSheet() {

// just clear sheet and set & style headings

var ss = SpreadsheetApp.getActiveSpreadsheet();

// set the tab to post - just for neatness

var sheetname = ss.getSheets()[0]setName('Posts');

// clear the sheet

var range = ss.getDataRange();

range.clear();

//set headings

ss.getRange('A1').setValue('Post ID');

ss.getRange('B1').setValue('Title');

ss.getRange('C1').setValue('Updated');

ss.getRange('A1:C1').setFontWeight('bold');

ss.getRange('A1:C1').setFontStyle('italic');

}

The main purpose of this function is to clear any data in the spreadsheet and then add headings to each column, in this case, keeping it simple, is just post id, post title and date modified. You can of course extend this as you like.

When you select the sub menu item it calls the function getPosts() as below

function getPosts(){
  setupSheet();
  // loop here as we need get back posts in chunks, Rest API max is 100 post
  offset = 0;
  per_page = 10;
  while ( (got=getPage(offset,per_page)) > 0 ) {
    offset+=got;
    Logger.log('offset %s', offset);
  }
}

function getPosts(){

setupSheet();

// loop here as we need get back posts in chunks, Rest API max is 100 post

offset = 0;

per_page = 10;

while ( (got=getPage(offset,per_page)) > 0 ) {

offset+=got;

Logger.log('offset %s', offset);

}

This does two things, first it clears the sheet ( again ) and then it loops through ‘pages’ of posts. The WP REST API limits the amount of post that can be retrieved in a single call to 100, and the default is 10. The limit is there to protect your website, but it does means we need to loop multiple times and keep track of what we have got. So we start with and ‘offset’ of zero and call the getPage() function until getPage() returns zero, if getPage() returns a +ve number ‘got’ we add that to the offset and go again.

The getPage() function does the API work

function getPage(offset,per_page) {
  // gets posts in chunks of per_page
  var ui = SpreadsheetApp.getUi();  // used for error messages
  var ss = SpreadsheetApp.getActiveSpreadsheet();
  
  var options = {
      'method': 'get',
      "contentType" : "application/json",
      'muteHttpExceptions': true
    }
  var apiHost = 'http://demo.wp-api.org/wp-json';   // set to your own domain
  url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;
  try {
    var response = UrlFetchApp.fetch(url, options);
    var data = JSON.parse(response)
    // loop through the map and output to sheet
    for (i = 0; i < data.length; i++) {
     row=offset+i+2;  // set the row to make sure it is below header and takes into account the paging
     ss.getRange('A'+row).setValue(data[i].id);
     ss.getRange('B'+row).setValue(data[i].title.rendered);
     ss.getRange('C'+row).setValue(data[i].modified);
  }
    return data.length;
  } catch(error) {
    var result = ui.alert( error.toString());
  }
  return 0;
}

function getPage(offset,per_page) {

// gets posts in chunks of per_page

var ui = SpreadsheetApp.getUi(); // used for error messages

var ss = SpreadsheetApp.getActiveSpreadsheet();

var options = {

'method': 'get',

"contentType" : "application/json",

'muteHttpExceptions': true

}

var apiHost = 'http://demo.wp-api.org/wp-json'; // set to your own domain

url = apiHost + '/wp/v2/posts?per_page='+per_page+'&offset='+offset;

try {

var response = UrlFetchApp.fetch(url, options);

var data = JSON.parse(response)

// loop through the map and output to sheet

for (i = 0; i < data.length; i++) {

row=offset+i+2; // set the row to make sure it is below header and takes into account the paging

ss.getRange('A'+row).setValue(data[i].id);

ss.getRange('B'+row).setValue(data[i].title.rendered);

ss.getRange('C'+row).setValue(data[i].modified);

}

return data.length;

} catch(error) {

var result = ui.alert( error.toString());

}

return 0;

}

The key call is var response = UrlFetchApp.fetch(url, options); which pull back from the ‘url’ based on the ‘options’ which in this example is http://demo.wp-api.org/wp-json/wp/v2/posts with query args of per_page and offset ( as you recall we increment offset ) and the function returns the number of rows – you may think that is always 10, but that isn’t always so, for instance 14 posts is one page of 10 and one page of 4.

Once we get the response we use JSON.parse(response) to build a map ( array for PHPers) of the posts and we loop through those and push them into the appropriate row & column, taking into consideration the header row and the offset.

In my next posts I am going to cover

How to get meta data, if you inspect what you get back you will find meta[] by default, and meta data is most useful -> post is here
How to authenticate with your user and password so you can do updates, and some gotchas of security software -> post is here
And tie it all together with an example script to retrieve posts, with meta data and apply updates -> post is here

I have also built a script to handle Woocommerce regular-price updates based on the principals in these tutorials, if there is interest in the write up – or even as a straight add-on please leave a comment.

As it does take time to write these post I would appreciate any feedback, encouragement or comments

5 minute guide to setup process monitoring with MONIT on Linux

30th August 2016 by Alan Leave a Comment

If you are running a VPS, for a production server, it is very useful to have a process that monitors, and restarts if failed, and notifies you of key processes, like apache, mysql, postfix & ssh.

Monit is a free open source tool that does the job.

Took me a little while to get my head around the setup, but it is actually very simple, one you know, and can be done in 5 minutes.

These are the step I take on Debian (should be the same for Ubuntu)

sudo apt-get install monit

1	sudo apt-get install monit

You don’t need to start anything or add it to any startup files, it does that automatically

There are a few settings needed to allow notification, and email address, an email server and a httpd server so you can monitor. These are in /etc/monit/monitrc

sudo vim /etc/monit/monitrc

1	sudo vim /etc/monit/monitrc

Find the comment line with the word mail server. My mail-server is on MailGun and my server is on Google Cloud Compute so uses port 2525. (obviously change the myuser / mypasswd / mydomain )

set mailserver smtp.mailgun.org port 2525 
    username "myuser@mydomain.com" 
    password "mypasswd"

set mailserver smtp.mailgun.org port 2525

username "myuser@mydomain.com"

password "mypasswd"

Find the comment line # set alert sysadm@foo.bar and add the line

set alert myuser@mydomain.com

1	set alert myuser@mydomain.com

Find the comment # set httpd port 2812 and uncomment (remove the #) the first 3 lines so you get

set httpd port 2812 
   and use address localhost # only accept connection from localhost 
   allow localhost # allow localhost to connect to the server and

set httpd port 2812

and use address localhost # only accept connection from localhost

allow localhost # allow localhost to connect to the server and

Save the file & then do a quick configuration test

sudo monit -t

1	sudo monit -t

if OK reload monit

sudo /etc/init.d/monit reload

1	sudo /etc/init.d/monit reload

and check the log (as any issues with mail server will show now )

sudo tail -f /var/log/monit.log

1	sudo tail -f /var/log/monit.log

ctril-c to stop following the log

check the status, the following command should show just one ‘System’

sudo monit status

1	sudo monit status

Now we add the services we want to monitor, for me that is apache, mysql, ssh and postfix and these are all redefined in /etc/monit/monitrc.d so we just need to copy the definitions into /etc/monit/conf.d like so

cd /etc/monit/monitrc.d
sudo cp apache2 openssh-server mysql postfix ../conf.d

1 2	cd /etc/monit/monitrc.d sudo cp apache2 openssh-server mysql postfix ../conf.d

restart monit

sudo /etc/init.d/monit reload

1	sudo /etc/init.d/monit reload

check the status , the following should give you all the info

sudo monit status |more

1	sudo monit status \|more

And that is it, now if say mysql dies, it will be restarted and you will get an email

Adding Swap to Google Compute Engine

15th August 2016 by Alan Leave a Comment

By default Google Compute Engine (Debian) doesn’t come with any swap space. If you have periodic programs that eat a bit more memory (like my backups do) then instead on just adding more physical memory you can add a swap file.

Swaps files are much slower than physical memory, so should be avoided except when it is needed.

Setting up a swap file on Debian on Google Compute Engine is described here

sudo fallocate -l 2G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
sudo swapon -s

sudo fallocate -l 2G /swapfile

sudo chmod 600 /swapfile

sudo mkswap /swapfile

sudo swapon /swapfile

sudo swapon -s

To make this permanent add this to /etc/fstab

/swapfile none swap sw 0 0

1	/swapfile none swap sw 0 0

If you need to resize the swap file

sudo swapoff /swapfile 
sudo fallocate -l 4G /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile

sudo swapoff /swapfile

sudo fallocate -l 4G /swapfile

sudo mkswap /swapfile

sudo swapon /swapfile

How much swap? Well that is down to your system. Maybe start with 1x your physical memory and then run memory diagnostics tools.

Backing up to gcloud storage from Linux/Virtualmin

5th August 2016 by Alan Leave a Comment

Using Google Cloud Storage as backup for your linux server is inexpensive if you use the nearline storage option. This is cheaper that Amazon Web Services (at the time of writing)

The process I use is as follows.

Backup locally
Create storage bucket / folders, set lifecycle
Script to move backups
find way to execute script

As I use Virtualmin as my control panel, the process details they way to use it from Virtualmin, but you can tailor to your own needs.

I’m assuming you have already signed up to Google Cloud Console and set up a project / billing as required.

The process uses a local backup copy and then moves / deletes it. So if you are limited in working space, you may need to divide you backups in to workable tranches.

You will need gstutils loaded load onto your server using the Google SDK https://cloud.google.com/sdk/docs/ and properly authorised – e.g. via gcloud init

1. Using virtualmin to create scheduled backups

e.g. creating backups into the local directory /home/backups/files

2. Make a Google Storage Bucket

You can use the Google Cloud Console to do this, but as you have the gsutil command and you will need to get used to it, here is how to do it from the CLI.

I’m assuming assuming here you want to store in EU but you can use US or ASIA as required for -l (location)

I recommend the ‘class’ to be ‘nearline’ as this is the cheaper option -c nl

(change mybucket to a unique name of course)

gsutil mb -c nl -l EU gs://mybucket/

1	gsutil mb -c nl -l EU gs://mybucket/

set life cycle e.g. 35 days to delete or as required, that way your storage bills won’t grow forever.

create rule file as below (35 days) and run

gsutil lifecycle set lifecycle_config.json gs://mybucket/

1	gsutil lifecycle set lifecycle_config.json gs://mybucket/

{
    "rule":
    [
      {
        "action": {"type": "Delete"},
        "condition": {"age":35 }
      }
    ]
}

{

"rule":

[

{

"action": {"type": "Delete"},

"condition": {"age":35 }

}

]

}

3. create script in /files/backups

#!/bin/bash
#today variable as YYYY-MM-DD
NOW=$(date +"%Y-%m-%d")
#Upload all files to our dated folder in the selected bucket
gsutil cp /home/backups/files/* gs://mybuckets/$1/$NOW
#Delete all the files after uploading them
rm /home/backups/files/*

#!/bin/bash

#today variable as YYYY-MM-DD

NOW=$(date +"%Y-%m-%d")

#Upload all files to our dated folder in the selected bucket

gsutil cp /home/backups/files/* gs://mybuckets/$1/$NOW

#Delete all the files after uploading them

rm /home/backups/files/*

Takes a folder argument allows different backups to use it within the same bucket (folders get created automatically)

e.g. /bin/sh /home/backups/upload.sh weekly

make sure your script is executable

sudo chmod +x /home/backups/upload.sh

1	sudo chmod +x /home/backups/upload.sh

4. find ways to execute script

With Virualmin this is easy, in schedule set command to run after backup

Recovering backups

Of course to recover backs requires copying locally and then restoring too – but with gsutil that is fairly easy.

list your weekly backups

gsutil ls gs://mybucket/weekly

1	gsutil ls gs://mybucket/weekly

e.g.

gs://mybucket/weekly/

gs://mybucket/weekly/2016-07-31

gs://mybucket/weekly/2016-07-25/

gs://mybucket/weekly/2016-08-05/

get the full directory (copy recursively) for 2016-08-05 back into /home/backups/files

gsutil -m cp -r dir gs://mybucket/weekly/2016-08-05/ /home/backups/files

1	gsutil -m cp -r dir gs://mybucket/weekly/2016-08-05/ /home/backups/files

Then restore (in my case using Virtualmin) in the normal way

WordPress REST API retrieve data and email it as an xls using Google Sheets

Putting it all together – authenticate, read and update WordPress from a spreadsheet

How to authenticate WordPress Rest API from Google Sheets / Scripts

Mod Security

WordFence

Other Security Software

Getting post meta via WordPress REST API

Linking WordPress to a Spreadsheet using WP REST API and Google Sheets scripts

5 minute guide to setup process monitoring with MONIT on Linux

Adding Swap to Google Compute Engine

Backing up to gcloud storage from Linux/Virtualmin

1. Using virtualmin to create scheduled backups

2. Make a Google Storage Bucket

3. create script in /files/backups

4. find ways to execute script

Recovering backups

Categories

Subscribe to my Newsletter

Mod Security

WordFence

Other Security Software

1. Using virtualmin to create scheduled backups

2. Make a Google Storage Bucket

3. create script in /files/backups

4. find ways to execute script

Recovering backups

Categories

Tags

Subscribe to my Newsletter