It now does some server-side transcoding for stuff the PS3 doesn't have native support for like Vorbis audio, video codecs and iso images.
It's not perfect yet, but it works for most of the stuff I've got.
The mediatomb xml is pasted first, then the support scripts which handle the video transcoding, just don't forget to chmod +x them (they can be used standalone as well btw).
/etc/mediatomb/config.xml:
<?xml version="1.0" encoding="UTF-8"?>
<config version="2" xmlns="http://mediatomb.cc/config/2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://mediatomb.cc/config/2 http://mediatomb.cc/config/2.xsd">
<server>
<ui enabled="yes" show-tooltips="yes">
<accounts enabled="no" session-timeout="30">
<account user="mediatomb" password="mediatomb"/>
</accounts>
</ui>
<name>[NAME]</name>
<udn>uuid:dae13796-e78c-486c-ab54-f79c144a20f3</udn>
<home>[PATH_TO_HOME]</home>
<webroot>/usr/share/mediatomb/web</webroot>
<storage caching="yes">
<sqlite3 enabled="no">
<database-file>sqlite3.db</database-file>
</sqlite3>
<mysql enabled="yes">
<host>localhost</host>
<username>mediatomb</username>
<database>mediatomb</database>
</mysql>
</storage>
<protocolInfo extend="yes"/>
<custom-http-headers>
<add header="X-User-Agent: redsonic"/>
</custom-http-headers>
<manufacturerURL>redsonic.com</manufacturerURL>
<modelNumber>105</modelNumber>
<extended-runtime-options>
<ffmpegthumbnailer enabled="yes">
<thumbnail-size>128</thumbnail-size>
<seek-percentage>5</seek-percentage>
<filmstrip-overlay>yes</filmstrip-overlay>
<workaround-bugs>no</workaround-bugs>
<image-quality>8</image-quality>
</ffmpegthumbnailer>
<mark-played-items enabled="no" suppress-cds-updates="yes">
<string mode="prepend">*</string>
<mark>
<content>video</content>
</mark>
</mark-played-items>
</extended-runtime-options>
</server>
<import hidden-files="no">
<scripting script-charset="UTF-8">
<common-script>/usr/share/mediatomb/js/common.js</common-script>
<playlist-script>/usr/share/mediatomb/js/playlists.js</playlist-script>
<virtual-layout type="builtin">
<import-script>/usr/share/mediatomb/js/import.js</import-script>
</virtual-layout>
</scripting>
<mappings>
<extension-mimetype ignore-unknown="no">
<map from="mp3" to="audio/mpeg"/>
<map from="ogg" to="application/ogg"/>
<map from="asf" to="video/x-ms-asf"/>
<map from="asx" to="video/x-ms-asf"/>
<map from="wma" to="audio/x-ms-wma"/>
<map from="wax" to="audio/x-ms-wax"/>
<map from="wmv" to="video/x-ms-wmv"/>
<map from="wvx" to="video/x-ms-wvx"/>
<map from="wm" to="video/x-ms-wm"/>
<map from="wmx" to="video/x-ms-wmx"/>
<map from="m3u" to="audio/x-mpegurl"/>
<map from="pls" to="audio/x-scpls"/>
<map from="flv" to="video/x-flv"/>
<map from="avi" to="video/divx"/>
<map from="mkv" to="video/x-matroska"/>
<map from="mts" to="video/mpeg"/>
<map from="ts" to="video/mpeg"/>
<map from="m2ts" to="video/mpeg"/>
<map from="mov" to="video/x-quicktime"/>
<map from="vob" to="video/mpeg"/>
<map from="m4v" to="video/mp4"/>
<map from="vob" to="video/iso"/>
<map from="VOB" to="video/iso"/>
<map from="iso" to="video/iso"/>
<map from="ISO" to="video/iso"/>
<map from="img" to="video/iso"/>
<map from="IMG" to="video/iso"/>
<map from="rm" to="video/realmedia"/>
<map from="rmvb" to="video/realmedia"/>
</extension-mimetype>
<mimetype-upnpclass>
<map from="audio/*" to="object.item.audioItem.musicTrack"/>
<map from="video/*" to="object.item.videoItem"/>
<map from="image/*" to="object.item.imageItem"/>
</mimetype-upnpclass>
<mimetype-contenttype>
<treat mimetype="audio/mpeg" as="mp3"/>
<treat mimetype="application/ogg" as="ogg"/>
<treat mimetype="audio/x-flac" as="flac"/>
<treat mimetype="image/jpeg" as="jpg"/>
<treat mimetype="audio/x-mpegurl" as="playlist"/>
<treat mimetype="audio/x-scpls" as="playlist"/>
<treat mimetype="audio/x-wav" as="pcm"/>
<treat mimetype="video/x-msvideo" as="avi"/>
<treat mimetype="video/quicktime" as="mov"/>
<treat mimetype="video/x-quicktime" as="mov"/>
</mimetype-contenttype>
</mappings>
<online-content>
<YouTube enabled="no" refresh="28800" update-at-start="no" purge-after="604800" racy-content="exclude" format="mp4" hd="no">
<favorites user="mediatomb"/>
<standardfeed feed="most_viewed" time-range="today"/>
<playlists user="mediatomb"/>
<uploads user="mediatomb"/>
<standardfeed feed="recently_featured" time-range="today"/>
</YouTube>
</online-content>
</import>
<transcoding enabled="yes">
<mimetype-profile-mappings>
<transcode mimetype="video/x-matroska" using="transcode-video"/>
<transcode mimetype="video/x-flv" using="transcode-video"/>
<transcode mimetype="video/x-quicktime" using="transcode-video"/>
<transcode mimetype="video/realmedia" using="transcode-video"/>
<transcode mimetype="application/ogg" using="vlcmpeg"/>
<transcode mimetype="application/ogg" using="oggflac2raw"/>
<transcode mimetype="audio/x-flac" using="oggflac2raw"/>
<transcode mimetype="video/iso" using="video-iso"/>
</mimetype-profile-mappings>
<profiles>
<profile name="transcode-video" enabled="yes" type="external">
<avi-fourcc-list mode="ignore">
<fourcc>DX50</fourcc>
<fourcc>DM4V</fourcc>
<fourcc>M4S2</fourcc>
</avi-fourcc-list>
<mimetype>video/mpeg</mimetype>
<accept-url>yes</accept-url>
<first-resource>yes</first-resource>
<hide-original-resource>yes</hide-original-resource>
<accept-ogg-theora>yes</accept-ogg-theora>
<agent command="/usr/local/bin/transcode-video" arguments="%in %out"/>
<buffer size="1048576" chunk-size="26214" fill-size="52428"/>
</profile>
<profile name="video-iso" enabled="yes" type="external">
<mimetype>video/mpeg</mimetype>
<first-resource>yes</first-resource>
<hide-original-resource>yes</hide-original-resource>
<accept-ogg-theora>yes</accept-ogg-theora>
<agent command="/usr/local/bin/transcode-video-iso" arguments="%in %out"/>
<buffer size="1048576" chunk-size="131072" fill-size="262144"/>
</profile>
<profile name="oggflac2raw" enabled="no" type="external">
<mimetype>audio/L16</mimetype>
<accept-url>no</accept-url>
<first-resource>yes</first-resource>
<accept-ogg-theora>no</accept-ogg-theora>
<agent command="ogg123" arguments="-d raw -f %out %in"/>
<buffer size="1048576" chunk-size="131072" fill-size="262144"/>
</profile>
<profile name="vlcmpeg" enabled="yes" type="external">
<mimetype>video/mpeg</mimetype>
<accept-url>yes</accept-url>
<first-resource>yes</first-resource>
<accept-ogg-theora>yes</accept-ogg-theora>
<agent command="vlc" arguments="-I dummy %in --sout #transcode{venc=ffmpeg,vcodec=mp2v,vb=4096,fps=25,aenc=ffmpeg,acodec=mpga,ab=192,samplerate=44100,channels=2}:standard{access=file,mux=ps,dst=%out} vlc:quit"/>
<buffer size="14400000" chunk-size="512000" fill-size="120000"/>
</profile>
</profiles>
</transcoding>
</config>
/usr/local/bin/transcode-video:
#!/bin/bash
exec "/usr/bin/avconv" -i "$1" -vcodec mpeg2video -b 4096k -r 25 -acodec ac3 -ar 48000 -ac 2 -ab 448000 -f mpegts - > "$2"
/usr/local/bin/transcode-video-iso:
#!/bin/bash
exec "/usr/bin/vlc" -v dvdsimple://"$1" -I dummy --sout "#transcode{vcodec=mpgv,vb=800,fps=25,acodec=mpga,ab=192,samplerate=48000,channels=2,deinterlace,soverlay,audio-sync}:standard{mux=ps,access=file,dst=$2" --sub-language=en vlc:quit >> /var/log/transcode.log 2>&1