Add custom script tidy_page_cache.sh

Berksfile.lock

@@ -5,5 +5,5 @@ DEPENDENCIES
 
 GRAPH
   apache2 (5.2.1)
-  foswiki (2.6.1)
+  foswiki (2.7.0)
     apache2 (< 6.0)

README.md

@@ -147,6 +147,9 @@ foswiki_patches name do
   owner             # owner of patched files
   group             # used group for patched files
 end
+```
+
+Moreover if `['foswiki']['config']['Cache']['Enabled']` is set to `true`, a script called `tidy_page_cache.sh` is installed under the tools directory. This can be used to remove orphaned or unnecessary entries. With `['foswiki']['cron']['tidy_page_cache_time']` the script can be run regularly.
 
 ## robots.txt
 

attributes/default.rb

@@ -16,11 +16,14 @@ default['foswiki']['cron']['mailto'] = 'root'
 default['foswiki']['cron']['mailto'] = 'root'
 default['foswiki']['cron']['statistics_time'] = '0 0 * * *'
 default['foswiki']['cron']['tick_time'] = '0 0 * * 0'
+default['foswiki']['cron']['tidy_page_cache_max_age'] = '1 year'
+default['foswiki']['cron']['tidy_page_cache_time'] = nil
 default['foswiki']['cron']['trash_cleanup_time'] = nil
 
 # Attributes set by tools/configure
-default['foswiki']['config']['Cache']['Implementation'] = 'Foswiki::PageCache::DBI::Generic'
+default['foswiki']['config']['Cache']['DBI']['TablePrefix'] = 'foswiki_cache'
 default['foswiki']['config']['Cache']['Enabled'] = false
+default['foswiki']['config']['Cache']['Implementation'] = 'Foswiki::PageCache::DBI::Generic'
 default['foswiki']['config']['DataDir'] = '{install}/data'
 default['foswiki']['config']['DefaultUrlHost'] = 'http://localhost'
 default['foswiki']['config']['Htpasswd']['FileName'] =

kitchen.yml

@@ -46,3 +46,53 @@ suites:
           disallow:
             - /foswiki/bin/view/Extensions
             - /Extensions
+  - name: foswiki-mysql
+    run_list:
+      - recipe[foswiki]
+      - recipe[foswiki::apache]
+    verifer:
+      inspec_tests:
+        - test/integration/default
+    attributes:
+      apache:
+        listen: ['*:2406']
+      foswiki:
+        apache:
+          http_port: 2406
+          server_admin: www-data@localhost
+          server_alias: wiki.localhost
+          server_name: localhost
+        config:
+          Cache:
+            DBI:
+              MySQL:
+                Database: foswiki
+                Host: localhost
+                Password: password
+                Username: foswiki
+            Enabled: true
+            Implementation: Foswiki::PageCache::DBI::MySQL
+        cron:
+          tidy_page_cache_time: '*/10 * * * *'
+  - name: foswiki-sqlite
+    run_list:
+      - recipe[foswiki]
+      - recipe[foswiki::apache]
+    verifer:
+      inspec_tests:
+        - test/integration/default
+    attributes:
+      apache:
+        listen: ['*:2406']
+      foswiki:
+        config:
+          apache:
+            http_port: 2406
+            server_admin: www-data@localhost
+            server_alias: wiki.localhost
+            server_name: localhost
+          Cache:
+            Enabled: true
+            Implementation: Foswiki::PageCache::DBI::SQLite
+        cron:
+          tidy_page_cache_time: '*/10 * * * *'

metadata.rb

@@ -3,7 +3,7 @@ maintainer       'HPC'
 maintainer_email 'hpc@gsi.de'
 license          'All rights reserved'
 description      'Installs/Configures Foswiki'
-version          '2.6.1'
+version          '2.7.0'
 depends          'apache2', '< 6.0'
 supports         'debian'
 supports         'ubuntu'

recipes/_cache.rb

@@ -9,14 +9,20 @@
 
 Chef::Recipe.include Foswiki::Helper
 
-case node['foswiki']['config']['Cache']['Implementation']
-when 'Foswiki::PageCache::DBI::SQLite'
-  package 'libdbd-sqlite3-perl'
+username = 'foswiki'
+password = ''
+database = 'foswiki'
+
+file "#{foswiki_dir('tools')}/mysql.cnf" do
+  action :delete
+end
 
+case node['foswiki']['config']['Cache']['Implementation']
 when 'Foswiki::PageCache::DBI::MySQL'
   package 'libdbd-mysql-perl'
 
   hostname = node['foswiki']['config']['Cache']['DBI']['MySQL']['Host']
+  port = node['foswiki']['config']['Cache']['DBI']['MySQL']['Port'] || 3306
   if hostname.eql?('localhost') || hostname.eql?('127.0.0.1')
     package 'default-mysql-server'
 
@@ -24,16 +30,50 @@ when 'Foswiki::PageCache::DBI::MySQL'
     password = foswiki_secret(node['foswiki']['config']['Cache']['DBI']['MySQL']['Password']).dup
     database = foswiki_secret(node['foswiki']['config']['Cache']['DBI']['MySQL']['Database'])
 
-    password.gsub!('$', '\\$')
-    password.gsub!('`', '\\`')
+    escaped_password = password.gsub('$', '\\$')
+    escaped_password.gsub!('`', '\\`')
 
     execute 'initialize page cache database' do
       command "echo \"CREATE DATABASE IF NOT EXISTS \\`#{database}\\` CHARACTER SET = 'utf8'; " \
         "GRANT ALL PRIVILEGES ON \\`#{database}\\`.* TO '#{username}'@" \
-        "'#{hostname}' IDENTIFIED BY '#{password}';\" | mysql -u root"
+        "'#{hostname}' IDENTIFIED BY '#{escaped_password}';\" | mysql -u root"
       sensitive true
     end
+
   else
     package 'default-mysql-client'
   end
+
+  q = password.include?('"') ? "'" : '"'
+  edit_resource :file, "#{foswiki_dir('tools')}/mysql.cnf" do
+    content "[mysql]\nhost=#{hostname}\nport=#{port}\nuser=#{username}\npassword=#{q}#{password}#{q}\n"
+    mode '0440'
+    owner node['apache']['user']
+    group node['apache']['group']
+    sensitive true
+    action :create
+  end
+
+when 'Foswiki::PageCache::DBI::SQLite'
+  package 'libdbd-sqlite3-perl'
+
+  database = node['foswiki']['config']['Cache']['DBI']['SQLite']['Filename'] ||
+             "#{foswiki_dir('working')}/sqlite.db"
+
+end
+
+template "#{foswiki_dir('tools')}/tidy_page_cache.sh" do
+  source 'tidy_page_cache.sh.erb'
+  variables(
+    cache_dir: "#{foswiki_dir('working')}/cache",
+    cache_implementation: node['foswiki']['config']['Cache']['Implementation'],
+    database: database,
+    mysql_cnf: "#{foswiki_dir('tools')}/mysql.cnf",
+    data_dir: foswiki_dir('data'),
+    table_prefix: node['foswiki']['config']['Cache']['DBI']['TablePrefix']
+  )
+  mode '0770'
+  owner node['apache']['user']
+  group node['apache']['group']
+  action :delete unless node['foswiki']['config']['Cache']['Enabled']
 end

recipes/default.rb

@@ -185,6 +185,10 @@ template '/etc/cron.d/foswiki' do
     script_dir: foswiki_dir('script'),
     statistics_time: node['foswiki']['cron']['statistics_time'],
     tick_time: node['foswiki']['cron']['tick_time'],
+    tidy_page_cache_max_age: node['foswiki']['cron']['tidy_page_cache_max_age'],
+    tidy_page_cache_time: if node['foswiki']['config']['Cache']['Enabled']
+                            node['foswiki']['cron']['tidy_page_cache_time']
+                          end,
     tools_dir: foswiki_dir('tools'),
     trash_cleanup_time: node['foswiki']['cron']['trash_cleanup_time'],
     user: node['apache']['user']

templates/foswiki-cron.erb

@@ -16,6 +16,9 @@
 <% unless @tick_time.nil? -%>
 <%= @tick_time %> <%= @user %> cd "<%= @script_dir %>" && perl ../tools/tick_foswiki.pl
 <% end -%>
+<% unless @tidy_page_cache_time.nil? -%>
+<%= @tidy_page_cache_time %> <%= @user %> '<%= @tools_dir %>/tidy_page_cache.sh' <%= @tidy_page_cache_max_age.nil? ? '' : "'#{@tidy_page_cache_max_age}'" %> >/dev/null
+<% end -%>
 <% unless @trash_cleanup_time.nil? -%>
 <%= @trash_cleanup_time %> <%= @user %> cd "<%= @script_dir %>" && ./rest /TrashPlugin/cleanUp
 <% end -%>

templates/tidy_page_cache.sh.erb

+#!/bin/bash
+set -o errexit -o nounset
+
+CACHE_DIR='<%= @cache_dir %>'
+DATA_DIR='<%= @data_dir %>'
+INTERVAL=
+TABLE_PREFIX='<%= @table_prefix %>'
+
+UNIMPORTANT_VARIATIONS=(raw rev sortcol)
+UNIMPORTANT_TOPICS=('%.WebCreateNewTopic')
+
+# Test environment
+for dir in "$CACHE_DIR" "$DATA_DIR"; do
+	[ -d "$dir" ] && continue
+	echo "$dir does not exist" >&2
+	exit 1
+done
+if [ $# -gt 0 ]; then
+	INTERVAL="$1"
+	if ! echo "$INTERVAL" | grep -Eq '^-?[0-9]+\s+[A-Za-z_]+$'; then
+		echo "Wrong interval format: $INTERVAL" >&2
+		exit 1
+	fi
+fi
+
+cached_topics() {
+	echo "SELECT DISTINCT topic, md5 FROM ${TABLE_PREFIX}_pages $*" | wiki_sql
+}
+count_cached_topics() {
+	echo "SELECT DISTINCT COUNT(topic) FROM ${TABLE_PREFIX}_pages" | wiki_sql
+}
+tell_status() { # job_description
+	after_count=$(count_cached_topics)
+	echo "Kept $after_count topics from $before_count after $*"
+	before_count="$after_count"
+}
+wiki_sql() {
+<% if @cache_implementation.eql? 'Foswiki::PageCache::DBI::MySQL' -%>
+	mysql --defaults-file='<%= @mysql_cnf %>' --skip-column-names '<%= @database %>'
+<% elsif @cache_implementation.eql? 'Foswiki::PageCache::DBI::SQLite' -%>
+	while read -r cmd; do
+		sqlite3 '<%= @database %>' "$cmd"
+	done
+<% else -%>
+	echo 'Invalid cache implementation: <%= @cache_implementation %>' >&2
+	exit 10
+<% end -%>
+}
+
+before_count=$(count_cached_topics)
+
+# Expire old entries
+if [ -n "$INTERVAL" ]; then
+	echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE STR_TO_DATE(lastmodified, '%a, %d %b %Y') < DATE_SUB(curdate(), interval $INTERVAL)" | wiki_sql
+	tell_status 'expiring old entries'
+fi
+
+# Remove unimportant entries
+{
+	for var in "${UNIMPORTANT_VARIATIONS[@]}"; do
+		echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE variation LIKE '%::$var=%';"
+	done
+	for topic in "${UNIMPORTANT_TOPICS[@]}"; do
+		echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE topic LIKE '$topic';"
+	done
+} | wiki_sql
+tell_status 'removing unimportant entries'
+
+# Remove non-existing topics
+cached_topics | while read -r topic md5; do
+	[ -f "$DATA_DIR/${topic//.//}.txt" ] && continue
+	rm /srv/foswiki/working/cache/$md5 >/dev/null 2>&1 || true
+	echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE topic = '$topic';"
+	echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE from_topic = '$topic';"
+done | wiki_sql
+tell_status 'removing non-existing topics'
+
+# Remove topics without cache files
+cached_topics | while read -r topic md5; do
+	[ -f "$CACHE_DIR/$md5" ] && continue
+	echo "A cache file for $topic has vanished" >&2
+	echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE topic = '$topic' AND md5 = '$md5';"
+done | wiki_sql
+tell_status 'removing topics with vanished cache files'
+
+# Clear dependencies
+before_count=$(echo "SELECT COUNT(*) FROM ${TABLE_PREFIX}_deps" | wiki_sql)
+{
+	for var in "${UNIMPORTANT_VARIATIONS[@]}"; do
+		echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE variation LIKE '%::$var=%';"
+	done
+	for topic in "${UNIMPORTANT_TOPICS[@]}"; do
+		echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE from_topic LIKE '$topic';"
+	done
+	echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE from_topic NOT IN (SELECT DISTINCT topic FROM ${TABLE_PREFIX}_pages);"
+	echo "DELETE deps FROM ${TABLE_PREFIX}_deps AS deps LEFT OUTER JOIN ${TABLE_PREFIX}_pages AS pages ON topic = from_topic AND deps.variation = pages.variation WHERE topic IS NULL;"
+} | wiki_sql
+after_count=$(echo "SELECT COUNT(*) FROM ${TABLE_PREFIX}_deps" | wiki_sql)
+echo "Kept $after_count dependencies from $before_count"
+
+# Remove unused cache files
+start_cache_size=$(du -sh "$CACHE_DIR" | awk '{print $1}')
+find /srv/foswiki/working/cache/ -type f -exec basename {} \; |
+	sed 's/\(.*\)/SELECT COUNT(*), "\1" FROM '"$TABLE_PREFIX"'_pages WHERE md5 = "\1";/' |
+	wiki_sql | awk '$1==0{print $2}' |
+	while read -r md5; do
+		rm "$CACHE_DIR/$md5" >/dev/null 2>&1 || true
+	done
+end_cache_size=$(du -sh "$CACHE_DIR" | awk '{print $1}')
+echo "Kept $end_cache_size of cache files from $start_cache_size"