RenderToTexture reorients PlainColor animation

Hey there Python-Ogre's,

I'm using Python-Ogre to simulate 3d scenarios and output two * 2d camera views from the same camera position.
One view is the regular screen view. The other is a pixel mask of the animated characters in the scene.

I started with the Demo_SkeletalAnimattion.py demo code and added in a stripped down version of the SelectionBuffer sample http://wiki.pythonogre.org/index.php?title=CodeSnippets_Selection_Buffer&oldid=1622 from the Wiki that does Render to Texture while substituting PlainColor for the original object textures. This all seemed fine at first, but once I started saving the frame images, it became obvious that the animated 'jaquas' character were oriented wrongly in the PlainColor version of each frame. Specifically, the head of each animated jaquas was in the right place, but the feet were angles towards the center of the screen.
The is weird. It's not just a wrong step of the animation. They should never be on an angle like this.

I tried commenting out the use of the PlainColor MaterialSwitcher but still doing the render to texture.
This rendered everything in the right place again, but didn't have the plain color (of course).
This tells me that it is just the material substitution that is causing this.
How can this be?

It occurs to me that other Ogre coders should care about this because anybody using SelectionBuffer (as per Wiki) for detecting hits in an game would only be getting the right answer for head shots, but not for body/feet.

Here is the code derived from the SelectionBuffer sample.
I use this by constructing the SelectionBuffer in my FrameListener.__init__() and invoking self.selectionBuffer.dumpGroundTruth() in my FrameListener.frameEnded()

UPDATE: Cleaned up all non-contributing media, such that there are zero log errors, but the problem described still exists.
Full sample code is now included below.


# This code is in the Public Domain
# -----------------------------------------------------------------------------
# This source file is a simple prototype derived from the Python-Ogre demo
# 'Demo_SkeletalAnimation.py' and the SelectionBuffer sample code available
# on the Python-Ogre wiki.
#
# You may use this sample code for anything you like, it is not covered by the
# LGPL.
# -----------------------------------------------------------------------------
import sys
sys.path.insert(0,'..')
import PythonOgreConfig

import ogre.renderer.OGRE as ogre
import SampleFramework as sf
import operator

import ctypes as ctypes
import ImageColor

NUM_JAIQUAS = 2
mAnimationRotation = ogre.Degree(d=-360/NUM_JAIQUAS)
mAnimChop = 7.96666
mAnimChopBlend = 0.3
mAnimState=[0] * NUM_JAIQUAS
mAnimationSpeed=[0] * NUM_JAIQUAS
mSneakStartOffset=ogre.Vector3(100,200,30)
mSneakEndOffset=ogre.Vector3(0,0,0)
mOrientations=[0] * NUM_JAIQUAS
mBasePositions=[0] * NUM_JAIQUAS
mSceneNode=[0] * NUM_JAIQUAS
ANIMATION = "Sneak" # Run Sneak Stagger Turn Walk


#
# class to handle material switching without having to modify scene materials individually
# The idea is to substitute fixed colors for each object so we can produce ground truth masks
# with pixel level truth data for each separate object.
#
class MaterialSwitcher( ogre.MaterialManager.Listener ):
def __init__(self):
ogre.MaterialManager.Listener.__init__(self)
self.technique = ogre.MaterialManager.getSingleton().load("PlainColor", ogre.ResourceGroupManager.DEFAULT_RESOURCE_GROUP_NAME).getTechnique(0)

# takes into account that one Entity can have multiple SubEntities
def handleSchemeNotFound(self, index, name, material, lod, subEntity):
temp = str(type(subEntity))
if temp == "<class 'ogre.renderer.OGRE._ogre_.SubEntity'>":
if subEntity.materialName == "jaiqua":
subEntity.setCustomParameter(1, (1.0, 1.0, 1.0, 1.0))
else:
subEntity.setCustomParameter(1, (0.0, 0.0, 0.0, 0.0))

return self.technique
return None

#
# We need this attached to the depth target, otherwise we get problems with the compositor
# MaterialManager.Listener should NOT be running all the time - rather only when we're
# specifically rendering the target that needs it
#
class SelectionRenderListener(ogre.RenderTargetListener):
def __init__(self, materialListener):
ogre.RenderTargetListener.__init__(self)
self.materialListener = materialListener

def preRenderTargetUpdate(self, evt):
ogre.MaterialManager.getSingleton().addListener( self.materialListener )

def postRenderTargetUpdate(self, evt):
ogre.MaterialManager.getSingleton().removeListener( self.materialListener )


class SelectionBuffer():
def __init__(self, sceneManager, renderTarget, camera):
self.sceneMgr = sceneManager
self.camera = camera
self.renderTarget = renderTarget

self.texture = ogre.TextureManager.getSingleton().createManual("GroundTruthTex",
ogre.ResourceGroupManager.DEFAULT_RESOURCE_GROUP_NAME,
ogre.TEX_TYPE_2D,
self.renderTarget.getWidth(),
self.renderTarget.getHeight(),
0, ogre.PixelFormat.PF_R8G8B8, ogre.TU_RENDERTARGET)

self.renderTexture = self.texture.getBuffer().getRenderTarget()
self.renderTexture.setAutoUpdated(True)
self.renderTexture.setPriority(0)
self.renderTexture.addViewport( self.camera )
self.renderTexture.getViewport(0).setOverlaysEnabled(False)
self.renderTexture.getViewport(0).setClearEveryFrame(True)

# Comment out these four lines for no PlainColor substitution
self.materialSwitcher = MaterialSwitcher()
self.selectionRenderListener = SelectionRenderListener(self.materialSwitcher)
self.renderTexture.addListener( self.selectionRenderListener )
self.renderTexture.getViewport(0).setMaterialScheme("aa")

def dumpGroundTruth(self, frameNum):
path = 'c:\\temp\\TruthFrames\\frame%04d.png' % frameNum
self.renderTexture.writeContentsToFile(path)
print "Frame: %s" % path

class SkeletalApplication(sf.Application):

def goOneFrame(self):
"Starts the rendering loop. Show how to use the renderOneFrame Method"
if not self._setUp():
return
if self._isPsycoEnabled():
self._activatePsyco()

self.root.getRenderSystem()._initRenderTargets()
while True:
ogre.WindowEventUtilities().messagePump()
if not self.root.renderOneFrame(1.0/30.0):
break

def _createScene(self):
global NUM_JAIQUAS
global mAnimationRotation,mAnimChop,mAnimChopBlend,mAnimState,mAnimationSpeed,mSneakStartOffset
global mSneakEndOffset, mOrientations, mBasePositions, mSceneNode

sceneManager = self.sceneManager
camera = self.camera

#setup Shadows
sceneManager.setShadowTechnique(ogre.SHADOWTYPE_STENCIL_ADDITIVE) ## doesn't work on my laptop
sceneManager.setShadowTechnique(ogre.SHADOWTYPE_TEXTURE_MODULATIVE)
self.CurrentShadowCameraSetup = ogre.FocusedShadowCameraSetup()
if self.root.getRenderSystem().getCapabilities().hasCapability(ogre.RSC_HWRENDER_TO_TEXTURE):
## In D3D, use a 1024x1024 shadow texture
sceneManager.setShadowTextureSettings(1024, 2)
else:
## Use 512x512 texture in GL since we can't go higher than the window res
sceneManager.setShadowTextureSettings(512, 2)

sceneManager.setShadowColour(ogre.ColourValue(0.6, 0.6, 0.6))

# Setup animation default
ogre.Animation.setDefaultInterpolationMode(ogre.Animation.IM_LINEAR)
ogre.Animation.setDefaultRotationInterpolationMode(ogre.Animation.RIM_LINEAR)

# Need some basic light
sceneManager.AmbientLight = ogre.ColourValue(0.5, 0.5, 0.5)


# The jaiqua sneak animation doesn't loop properly, so lets hack it so it does
# We want to copy the initial keyframes of all bones, but alter the Spineroot
# to give it an offset of where the animation ends

## Doing this returns a SharedPtr_less_Ogre_scope_Resource_grate
self.skel = ogre.SkeletonManager.getSingleton().load("jaiqua.skeleton",
ogre.ResourceGroupManager.DEFAULT_RESOURCE_GROUP_NAME,
False,
ogre.ManualResourceLoader(),
ogre.NameValuePairList())


anim = self.skel.getAnimation(ANIMATION)

self.cameraNode = sceneManager.getRootSceneNode().createChildSceneNode()
self.cameraNode.attachObject(self.camera)

self.animation = sceneManager.createAnimation('TestTrack', 1)
self.animationTrack = self.animation.createNodeTrack(0, self.cameraNode)

blankKF = self.animationTrack.createKeyFrame (0)
evect = ogre.Vector3(0,0,0)
blankKF.setScale=evect
blankKF.setTranslate=evect


trackIter = anim.getNodeTrackIterator()
while (trackIter.hasMoreElements()):
track = trackIter.getNext()
oldKf = blankKF
track.getInterpolatedKeyFrame(mAnimChop, oldKf)

# Drop all keyframes after the chop
while (track.getKeyFrame(track.getNumKeyFrames()-1).getTime() >= mAnimChop - mAnimChopBlend):
track.removeKeyFrame(track.getNumKeyFrames()-1)

newKf = track.createNodeKeyFrame(mAnimChop)
startKf = track.getNodeKeyFrame(0)

bone = self.skel.getBone(track.getHandle())

if (bone.getName() == "Spineroot") :
mSneakStartOffset = startKf.getTranslate() + bone.getInitialPosition()
mSneakEndOffset = oldKf.getTranslate() + bone.getInitialPosition()
mSneakStartOffset.y = mSneakEndOffset.y
# Adjust spine root relative to new location
newKf.setRotation(oldKf.getRotation())
newKf.setTranslate(oldKf.getTranslate())
newKf.setScale(oldKf.getScale())
else:
newKf.setRotation(startKf.getRotation())
newKf.setTranslate(startKf.getTranslate())
newKf.setScale(startKf.getScale())

rotInc = ogre.Math.TWO_PI / NUM_JAIQUAS
rot = 0.0
for i in range( NUM_JAIQUAS) :
q = ogre.Quaternion()
q.FromAngleAxis(ogre.Radian(r=rot), ogre.Vector3().UNIT_Y)
mOrientations = q
mBasePositions = q * ogre.Vector3(0,0,-20)
ent = sceneManager.createEntity("jaiqua" + str(i), "jaiqua.mesh")

# Add entity to the scene node
mSceneNode = sceneManager.getRootSceneNode().createChildSceneNode()
mSceneNode.attachObject(ent)
mSceneNode.rotate(q)
mSceneNode.translate(mBasePositions)

mAnimState = ent.getAnimationState(ANIMATION)
mAnimState.setEnabled(True)
mAnimState.setLoop(False) # manual loop since translation involved
mAnimationSpeed = ogre.Math.RangeRandom(0.5, 1.5)

rot = rot + rotInc

self.blueLight = sceneManager.createLight('BlueLight')
self.blueLight.setType(ogre.Light.LT_SPOTLIGHT)
self.blueLight.setPosition (-200, 150, -100)
dirvec = -self.blueLight.getPosition()
dirvec.normalise()
self.blueLight.setDirection(dirvec)
self.blueLight.setDiffuseColour(0.5, 0.5, 1.0)

self.greenLight = sceneManager.createLight('GreenLight')
self.greenLight.setType(ogre.Light.LT_SPOTLIGHT)
self.greenLight.setPosition (0, 150, -100)
dirvec = -self.greenLight.getPosition()
dirvec.normalise()
self.greenLight.setDirection(dirvec)
self.greenLight.setDiffuseColour (0.5, 1.0, 0.5)

self.TexCam = sceneManager.createCamera("ReflectCam")
self.TexCam.setCustomViewMatrix(False)

self.CurrentShadowCameraSetup.getShadowCamera(self.sceneManager, self.camera, self.viewport, self.blueLight, self.TexCam,0)
sceneManager.setShadowCameraSetup(self.CurrentShadowCameraSetup)

# Position the camera
self.camera.setPosition(100, 20, 0)
self.camera.lookAt(0, 10, 0)
self.camera.setFarClipDistance(100000)


self.plane = ogre.Plane()
self.plane.normal = ogre.Vector3().UNIT_Y
self.plane.d = 100
ogre.MeshManager.getSingleton().createPlane("Myplane",
ogre.ResourceGroupManager.DEFAULT_RESOURCE_GROUP_NAME, self.plane,
1500,1500,20,20,True,1,60,60,ogre.Vector3().UNIT_Z)
self.pPlaneEnt = sceneManager.createEntity( "plane", "Myplane" )
self.pPlaneEnt.setMaterialName("Examples/Rockwall")
self.pPlaneEnt.setCastShadows(False)
sceneManager.getRootSceneNode().createChildSceneNode(ogre.Vector3(0,99,0)).attachObject(self.pPlaneEnt)

def _createFrameListener(self):
self.frameListener = SkeletalAnimationFrameListener(self.sceneManager, self.renderWindow, self.camera )
self.root.addFrameListener(self.frameListener)

# def __del__ ( self ):
# self.sceneManager.destroyEntity(self.pPlaneEnt)
# sf.application.__del__(self)


class SkeletalAnimationFrameListener(sf.FrameListener):
global NUM_JAIQUAS
global mAnimationRotation,mAnimChop,mAnimChopBlend,mAnimState,mAnimationSpeed,mSneakStartOffset
global mSneakEndOffset, mOrientations, mBasePositions, mSceneNode

def __init__(self, sceneManager, renderWindow, camera ):
sf.FrameListener.__init__(self, renderWindow, camera)
self.sceneManager = sceneManager
self.frameNum = 0
self.selectionBuffer = SelectionBuffer(self.sceneManager, self.renderWindow, self.camera)

# Remove statistics overlay.
self.statisticsOn = False
self.showDebugOverlay(self.statisticsOn)

def frameStarted(self, frameEvent):
for i in range(NUM_JAIQUAS):
inc = frameEvent.timeSinceLastFrame * mAnimationSpeed
if (mAnimState.getTimePosition() + inc) >= mAnimChop :
# Reposition the scene node origin since animation includes translation
# Calculate as an offset to the end position, rotated by the
# amount the animation turns the character
rot = ogre.Quaternion(mAnimationRotation, ogre.Vector3().UNIT_Y)
startoffset = mSceneNode.getOrientation() * -mSneakStartOffset
endoffset = mSneakEndOffset
offset = rot * startoffset
currEnd = mSceneNode.getOrientation() * endoffset + mSceneNode.getPosition()
mSceneNode.setPosition(currEnd + offset)
mSceneNode.rotate(q=rot)
mAnimState.setTimePosition((mAnimState.getTimePosition() + inc) - mAnimChop)
else:
mAnimState.addTime(inc)

return sf.FrameListener.frameStarted(self, frameEvent)

def frameEnded(self, frameEvent):
# Dump frame numbered images to VideoFrames directory.
path = 'c:\\temp\\VideoFrames\\frame%04d.png' % self.frameNum
self.renderWindow.writeContentsToFile(path)

# Dump frame numbered ground truth to TruthFrames directory.
self.selectionBuffer.dumpGroundTruth(self.frameNum)

self.frameNum += 1

return sf.FrameListener.frameEnded(self, frameEvent)

if __name__ == '__main__':
try:
application = SkeletalApplication()
application.goOneFrame() # goOneFrame() makes animation live but non-RealTime.
except ogre.OgreException, e:
print e


PlainColor.cg and PlainColor.material are as per the Selection Buffer Wiki Article.

void main_plain_color_vp(
// Vertex Inputs
float4 position : POSITION, // Vertex position in model space
float2 texCoord0 : TEXCOORD0, // Texture UV set 0

// Outputs
out float4 oPosition : POSITION, // Transformed vertex position
out float2 uv0 : TEXCOORD0, // UV0

// Model Level Inputs
uniform float4x4 worldViewProj)
{
// Calculate output position
oPosition = mul(worldViewProj, position);

// Simply copy the input vertex UV to the output
uv0 = texCoord0;
}

void main_plain_color_fp(
// Pixel Inputs
float2 uv0 : TEXCOORD0, // UV interpolated for current pixel
// Outputs
out float4 color : COLOR, // Output color we want to write
uniform float4 inColor
)
{
color = inColor;
}


vertex_program PlainColor_VS cg
{
source PlainColor.cg
entry_point main_plain_color_vp
profiles vs_1_1 arbvp1

default_params
{
param_named_auto worldViewProj worldviewproj_matrix
}

}

fragment_program PlainColor_PS cg
{
source PlainColor.cg
entry_point main_plain_color_fp
profiles ps_1_1 arbfp1

default_params
{
param_named inColor float4 1 1 1 1
}
}

material PlainColor
{
// Material has one technique
technique
{
// This technique has one pass
pass
{
// Make this pass use the vertex shader defined above
vertex_program_ref PlainColor_VS
{
}
// Make this pass use the pixel shader defined above
fragment_program_ref PlainColor_PS
{
param_named_auto inColor custom 1
}
}
}
}



...AndrewD...

Well, I solved my own problem.
I expect anybody trying to use a technique like the SelectionBuffer demo from the wiki will want to do this too.
The problem was that the PlainColor vertex program (vp) code is not up to dealing with animations.
I made a hybrid jaiqua.material file with an extra technique for plain color rendering, that uses the Ogre/HardwareSkinningTwoWeights vertex program, but the PlainColor fragment_program.

I expect that if you want to do accurate SelectionBuffer processing for detecting selections or FPS shooting hits on animated targets, you will need to do something very similar to this too.

...AndrewD...

MaterialSwitcher now looks like:

class MaterialSwitcher( ogre.MaterialManager.Listener ):
def __init__(self):
ogre.MaterialManager.Listener.__init__(self)
self.generalPlainColorTechnique = ogre.MaterialManager.getSingleton().load("PlainColor", ogre.ResourceGroupManager.DEFAULT_RESOURCE_GROUP_NAME).getTechnique(0)

# takes into account that one Entity can have multiple SubEntities
def handleSchemeNotFound(self, index, name, material, lod, subEntity):
temp = str(type(subEntity))
if temp == "<class 'ogre.renderer.OGRE._ogre_.SubEntity'>":
if material.getName() == "jaiqua":
subEntity.setCustomParameter(1, (1.0, 1.0, 1.0, 1.0))
self.jaiquaPlainColorTechnique = material.getTechnique(2)
return self.jaiquaPlainColorTechnique

subEntity.setCustomParameter(1, (0.0, 0.0, 0.0, 0.0))
return self.generalPlainColorTechnique

return None



...and the jaiqas.material

fragment_program JaiquaPlainColor_PS cg
{
source PlainColor.cg
entry_point main_plain_color_fp
profiles ps_1_1 arbfp1

default_params
{
param_named inColor float4 1 1 1 1
}
}


material jaiqua
{
// Hardware skinning technique
technique
{
pass
{
vertex_program_ref Ogre/HardwareSkinningTwoWeights
{

}
// alternate shadow caster program
shadow_caster_vertex_program_ref Ogre/HardwareSkinningTwoWeightsShadowCaster
{
param_named_auto worldMatrix3x4Array world_matrix_array_3x4
param_named_auto viewProjectionMatrix viewproj_matrix
param_named_auto ambient ambient_light_colour

}

texture_unit
{
texture blue_jaiqua.jpg
tex_address_mode clamp
}
}
}

// Software blending technique
technique
{
pass
{
texture_unit
{
texture blue_jaiqua.jpg
tex_address_mode clamp
}
}
}

// PlainColor technique
technique plain
{
// This technique has one pass
pass
{
// Make this pass use the vertex shader defined above
vertex_program_ref Ogre/HardwareSkinningTwoWeights
{
}

// Make this pass use the pixel shader defined above
fragment_program_ref JaiquaPlainColor_PS
{
param_named_auto inColor custom 1
}
}
}

}