rajavijaysingh

The objective here is to find all SegmentNotFoundException type corruption and remove it (as we don't have a valid backup)

Before running any of the below steps we created an index.json file (see the file below) which contains only the index definitions of the corrupted indexes.

This generates the file indexing-results\index-definities.json

java -Xmx8g -jar oak-run-1.8.12.jar index --fds-path=crx-quickstart\repository\datastore crx-quickstart\repository\segmentstore --index-definitions

From that file we copied only the definitions we needed and created index.json (file attached here )

The latest version of this article now exists here instead: https://github.com/cqsupport/fix-instructions/blob/master/version-issues/

Old steps below:

Issue

When deleting versions from Apache Jackrabbit Oak you observe errors in the logs similar to the one below:

java.lang.NullPointerException: null

Breaklets

The Idea

Breaklets are a way to check limits of a platform/aspect/tool. Breaklets

... iteratively make things worse
... run until either
- a threshold is reached

	//Adaptation of @stillalex's script from here https://gist.github.com/stillalex/06303f8cc1d3780d3eab4c72575883ae
	//This version works with Oak 1.6 and later versions
	import java.io.InputStream;
	import java.util.concurrent.atomic.AtomicInteger
	import org.apache.jackrabbit.oak.api.Type
	import org.apache.jackrabbit.oak.spi.state.NodeState
	import org.apache.jackrabbit.oak.spi.state.NodeStore
	import org.apache.jackrabbit.oak.commons.PathUtils
	import com.google.common.collect.Lists
	import java.util.List

	# Output the access log
	cat access.log

	# Run the results through grep to find any 500 status codes
	cat access.log \| grep "[: ]500[: ]"

	# Find any non 200 status codes
	cat access.log \| grep -v "[: ]200[: ]"

	# Find multiple status codes

	{
	"Statement": [
	{
	"Action": [
	"apigateway:*",
	"cloudformation:CancelUpdateStack",
	"cloudformation:ContinueUpdateRollback",
	"cloudformation:CreateChangeSet",
	"cloudformation:CreateStack",
	"cloudformation:CreateUploadBucket",

	<?xml version="1.0" encoding="UTF-8"?>
	<workspaceFilter version="1.0">
	<filter root="/content">
	<exclude pattern="/content/jcr:system"/>
	<exclude pattern="/content/var/classes"/>
	<exclude pattern="^.*/rep:accessControl"/>
	</filter>
	</workspaceFilter>

	// gulpfile.js

	var namespace = 'gulp-flow';

	var gulp = require('gulp'),
	watch = require('gulp-watch'),
	gutil = require('gulp-util'),
	plumber = require('gulp-plumber'),
	slang = require('gulp-slang'),
	debug = require('gulp-debug'),

	<?xml version="1.0" encoding="UTF-8"?>
	<jcr:root xmlns:sling="http://sling.apache.org/jcr/sling/1.0"
	xmlns:jcr="http://www.jcp.org/jcr/1.0"
	xmlns:nt="http://www.jcp.org/jcr/nt/1.0"
	jcr:primaryType="nt:unstructured">
	<content jcr:primaryType="nt:unstructured">
	<items jcr:primaryType="nt:unstructured">
	<basic jcr:primaryType="nt:unstructured">
	<items jcr:primaryType="nt:unstructured">
	<column jcr:primaryType="nt:unstructured">

	Standard:
	-Xmx -- Increase heap size to 50% total machine ram, or 24gb -- whichever is smaller
	-XX:+UseG1GC -- More efficient server GC in Java 8
	-XX:+UseAES -XX:+UseAESIntrinsics -- Use Hardware crypto features for AES if available
	-XX:+AggressiveOpts -- Experimental faster JVM tunings
	-Djava.awt.headless=true -- Required for headless unix boxes for Java2D support to work properly
	-Djava.io.tmpdir=/tmp -- Recommended to set somewhere with fast i/o

	Debugging:
	-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -- Used for triaging heap full errors