Commit 8ec31683 authored by André Kerkhoff's avatar André Kerkhoff
Browse files

Add custom script tidy_page_cache.sh

parent 97a7ffa6
......@@ -5,5 +5,5 @@ DEPENDENCIES
GRAPH
apache2 (5.2.1)
foswiki (2.6.1)
foswiki (2.7.0)
apache2 (< 6.0)
......@@ -147,6 +147,9 @@ foswiki_patches name do
owner # owner of patched files
group # used group for patched files
end
```
Moreover if `['foswiki']['config']['Cache']['Enabled']` is set to `true`, a script called `tidy_page_cache.sh` is installed under the tools directory. This can be used to remove orphaned or unnecessary entries. With `['foswiki']['cron']['tidy_page_cache_time']` the script can be run regularly.
## robots.txt
......
......@@ -16,11 +16,14 @@ default['foswiki']['cron']['mailto'] = 'root'
default['foswiki']['cron']['mailto'] = 'root'
default['foswiki']['cron']['statistics_time'] = '0 0 * * *'
default['foswiki']['cron']['tick_time'] = '0 0 * * 0'
default['foswiki']['cron']['tidy_page_cache_max_age'] = '1 year'
default['foswiki']['cron']['tidy_page_cache_time'] = nil
default['foswiki']['cron']['trash_cleanup_time'] = nil
# Attributes set by tools/configure
default['foswiki']['config']['Cache']['Implementation'] = 'Foswiki::PageCache::DBI::Generic'
default['foswiki']['config']['Cache']['DBI']['TablePrefix'] = 'foswiki_cache'
default['foswiki']['config']['Cache']['Enabled'] = false
default['foswiki']['config']['Cache']['Implementation'] = 'Foswiki::PageCache::DBI::Generic'
default['foswiki']['config']['DataDir'] = '{install}/data'
default['foswiki']['config']['DefaultUrlHost'] = 'http://localhost'
default['foswiki']['config']['Htpasswd']['FileName'] =
......
......@@ -46,3 +46,53 @@ suites:
disallow:
- /foswiki/bin/view/Extensions
- /Extensions
- name: foswiki-mysql
run_list:
- recipe[foswiki]
- recipe[foswiki::apache]
verifer:
inspec_tests:
- test/integration/default
attributes:
apache:
listen: ['*:2406']
foswiki:
apache:
http_port: 2406
server_admin: www-data@localhost
server_alias: wiki.localhost
server_name: localhost
config:
Cache:
DBI:
MySQL:
Database: foswiki
Host: localhost
Password: password
Username: foswiki
Enabled: true
Implementation: Foswiki::PageCache::DBI::MySQL
cron:
tidy_page_cache_time: '*/10 * * * *'
- name: foswiki-sqlite
run_list:
- recipe[foswiki]
- recipe[foswiki::apache]
verifer:
inspec_tests:
- test/integration/default
attributes:
apache:
listen: ['*:2406']
foswiki:
config:
apache:
http_port: 2406
server_admin: www-data@localhost
server_alias: wiki.localhost
server_name: localhost
Cache:
Enabled: true
Implementation: Foswiki::PageCache::DBI::SQLite
cron:
tidy_page_cache_time: '*/10 * * * *'
......@@ -3,7 +3,7 @@ maintainer 'HPC'
maintainer_email 'hpc@gsi.de'
license 'All rights reserved'
description 'Installs/Configures Foswiki'
version '2.6.1'
version '2.7.0'
depends 'apache2', '< 6.0'
supports 'debian'
supports 'ubuntu'
......@@ -9,14 +9,20 @@
Chef::Recipe.include Foswiki::Helper
case node['foswiki']['config']['Cache']['Implementation']
when 'Foswiki::PageCache::DBI::SQLite'
package 'libdbd-sqlite3-perl'
username = 'foswiki'
password = ''
database = 'foswiki'
file "#{foswiki_dir('tools')}/mysql.cnf" do
action :delete
end
case node['foswiki']['config']['Cache']['Implementation']
when 'Foswiki::PageCache::DBI::MySQL'
package 'libdbd-mysql-perl'
hostname = node['foswiki']['config']['Cache']['DBI']['MySQL']['Host']
port = node['foswiki']['config']['Cache']['DBI']['MySQL']['Port'] || 3306
if hostname.eql?('localhost') || hostname.eql?('127.0.0.1')
package 'default-mysql-server'
......@@ -24,16 +30,50 @@ when 'Foswiki::PageCache::DBI::MySQL'
password = foswiki_secret(node['foswiki']['config']['Cache']['DBI']['MySQL']['Password']).dup
database = foswiki_secret(node['foswiki']['config']['Cache']['DBI']['MySQL']['Database'])
password.gsub!('$', '\\$')
password.gsub!('`', '\\`')
escaped_password = password.gsub('$', '\\$')
escaped_password.gsub!('`', '\\`')
execute 'initialize page cache database' do
command "echo \"CREATE DATABASE IF NOT EXISTS \\`#{database}\\` CHARACTER SET = 'utf8'; " \
"GRANT ALL PRIVILEGES ON \\`#{database}\\`.* TO '#{username}'@" \
"'#{hostname}' IDENTIFIED BY '#{password}';\" | mysql -u root"
"'#{hostname}' IDENTIFIED BY '#{escaped_password}';\" | mysql -u root"
sensitive true
end
else
package 'default-mysql-client'
end
q = password.include?('"') ? "'" : '"'
edit_resource :file, "#{foswiki_dir('tools')}/mysql.cnf" do
content "[mysql]\nhost=#{hostname}\nport=#{port}\nuser=#{username}\npassword=#{q}#{password}#{q}\n"
mode '0440'
owner node['apache']['user']
group node['apache']['group']
sensitive true
action :create
end
when 'Foswiki::PageCache::DBI::SQLite'
package 'libdbd-sqlite3-perl'
database = node['foswiki']['config']['Cache']['DBI']['SQLite']['Filename'] ||
"#{foswiki_dir('working')}/sqlite.db"
end
template "#{foswiki_dir('tools')}/tidy_page_cache.sh" do
source 'tidy_page_cache.sh.erb'
variables(
cache_dir: "#{foswiki_dir('working')}/cache",
cache_implementation: node['foswiki']['config']['Cache']['Implementation'],
database: database,
mysql_cnf: "#{foswiki_dir('tools')}/mysql.cnf",
data_dir: foswiki_dir('data'),
table_prefix: node['foswiki']['config']['Cache']['DBI']['TablePrefix']
)
mode '0770'
owner node['apache']['user']
group node['apache']['group']
action :delete unless node['foswiki']['config']['Cache']['Enabled']
end
......@@ -185,6 +185,10 @@ template '/etc/cron.d/foswiki' do
script_dir: foswiki_dir('script'),
statistics_time: node['foswiki']['cron']['statistics_time'],
tick_time: node['foswiki']['cron']['tick_time'],
tidy_page_cache_max_age: node['foswiki']['cron']['tidy_page_cache_max_age'],
tidy_page_cache_time: if node['foswiki']['config']['Cache']['Enabled']
node['foswiki']['cron']['tidy_page_cache_time']
end,
tools_dir: foswiki_dir('tools'),
trash_cleanup_time: node['foswiki']['cron']['trash_cleanup_time'],
user: node['apache']['user']
......
......@@ -16,6 +16,9 @@
<% unless @tick_time.nil? -%>
<%= @tick_time %> <%= @user %> cd "<%= @script_dir %>" && perl ../tools/tick_foswiki.pl
<% end -%>
<% unless @tidy_page_cache_time.nil? -%>
<%= @tidy_page_cache_time %> <%= @user %> '<%= @tools_dir %>/tidy_page_cache.sh' <%= @tidy_page_cache_max_age.nil? ? '' : "'#{@tidy_page_cache_max_age}'" %> >/dev/null
<% end -%>
<% unless @trash_cleanup_time.nil? -%>
<%= @trash_cleanup_time %> <%= @user %> cd "<%= @script_dir %>" && ./rest /TrashPlugin/cleanUp
<% end -%>
......
#!/bin/bash
set -o errexit -o nounset
CACHE_DIR='<%= @cache_dir %>'
DATA_DIR='<%= @data_dir %>'
INTERVAL=
TABLE_PREFIX='<%= @table_prefix %>'
UNIMPORTANT_VARIATIONS=(raw rev sortcol)
UNIMPORTANT_TOPICS=('%.WebCreateNewTopic')
# Test environment
for dir in "$CACHE_DIR" "$DATA_DIR"; do
[ -d "$dir" ] && continue
echo "$dir does not exist" >&2
exit 1
done
if [ $# -gt 0 ]; then
INTERVAL="$1"
if ! echo "$INTERVAL" | grep -Eq '^-?[0-9]+\s+[A-Za-z_]+$'; then
echo "Wrong interval format: $INTERVAL" >&2
exit 1
fi
fi
cached_topics() {
echo "SELECT DISTINCT topic, md5 FROM ${TABLE_PREFIX}_pages $*" | wiki_sql
}
count_cached_topics() {
echo "SELECT DISTINCT COUNT(topic) FROM ${TABLE_PREFIX}_pages" | wiki_sql
}
tell_status() { # job_description
after_count=$(count_cached_topics)
echo "Kept $after_count topics from $before_count after $*"
before_count="$after_count"
}
wiki_sql() {
<% if @cache_implementation.eql? 'Foswiki::PageCache::DBI::MySQL' -%>
mysql --defaults-file='<%= @mysql_cnf %>' --skip-column-names '<%= @database %>'
<% elsif @cache_implementation.eql? 'Foswiki::PageCache::DBI::SQLite' -%>
while read -r cmd; do
sqlite3 '<%= @database %>' "$cmd"
done
<% else -%>
echo 'Invalid cache implementation: <%= @cache_implementation %>' >&2
exit 10
<% end -%>
}
before_count=$(count_cached_topics)
# Expire old entries
if [ -n "$INTERVAL" ]; then
echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE STR_TO_DATE(lastmodified, '%a, %d %b %Y') < DATE_SUB(curdate(), interval $INTERVAL)" | wiki_sql
tell_status 'expiring old entries'
fi
# Remove unimportant entries
{
for var in "${UNIMPORTANT_VARIATIONS[@]}"; do
echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE variation LIKE '%::$var=%';"
done
for topic in "${UNIMPORTANT_TOPICS[@]}"; do
echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE topic LIKE '$topic';"
done
} | wiki_sql
tell_status 'removing unimportant entries'
# Remove non-existing topics
cached_topics | while read -r topic md5; do
[ -f "$DATA_DIR/${topic//.//}.txt" ] && continue
rm /srv/foswiki/working/cache/$md5 >/dev/null 2>&1 || true
echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE topic = '$topic';"
echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE from_topic = '$topic';"
done | wiki_sql
tell_status 'removing non-existing topics'
# Remove topics without cache files
cached_topics | while read -r topic md5; do
[ -f "$CACHE_DIR/$md5" ] && continue
echo "A cache file for $topic has vanished" >&2
echo "DELETE FROM ${TABLE_PREFIX}_pages WHERE topic = '$topic' AND md5 = '$md5';"
done | wiki_sql
tell_status 'removing topics with vanished cache files'
# Clear dependencies
before_count=$(echo "SELECT COUNT(*) FROM ${TABLE_PREFIX}_deps" | wiki_sql)
{
for var in "${UNIMPORTANT_VARIATIONS[@]}"; do
echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE variation LIKE '%::$var=%';"
done
for topic in "${UNIMPORTANT_TOPICS[@]}"; do
echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE from_topic LIKE '$topic';"
done
echo "DELETE FROM ${TABLE_PREFIX}_deps WHERE from_topic NOT IN (SELECT DISTINCT topic FROM ${TABLE_PREFIX}_pages);"
echo "DELETE deps FROM ${TABLE_PREFIX}_deps AS deps LEFT OUTER JOIN ${TABLE_PREFIX}_pages AS pages ON topic = from_topic AND deps.variation = pages.variation WHERE topic IS NULL;"
} | wiki_sql
after_count=$(echo "SELECT COUNT(*) FROM ${TABLE_PREFIX}_deps" | wiki_sql)
echo "Kept $after_count dependencies from $before_count"
# Remove unused cache files
start_cache_size=$(du -sh "$CACHE_DIR" | awk '{print $1}')
find /srv/foswiki/working/cache/ -type f -exec basename {} \; |
sed 's/\(.*\)/SELECT COUNT(*), "\1" FROM '"$TABLE_PREFIX"'_pages WHERE md5 = "\1";/' |
wiki_sql | awk '$1==0{print $2}' |
while read -r md5; do
rm "$CACHE_DIR/$md5" >/dev/null 2>&1 || true
done
end_cache_size=$(du -sh "$CACHE_DIR" | awk '{print $1}')
echo "Kept $end_cache_size of cache files from $start_cache_size"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment