UTF-8 basic support + international fonts

Resources/PakLocation.txt

@@ -1,9 +1,27 @@
 
-All paks for development vesion:
+
+All paks for development vesion
+-------------------------------
 
 http://yvt.jp/files/programs/osppaks/DevPaks17.zip
 
-(These paks are not included in the source tree because they contains some 
-non-opensource materials and they seem too large to include in the git tree.)
+These paks are not included in the source tree because they contains some 
+non-opensource materials and they seem too large to include in the git tree.
 
 
+
+M+ BITMAP FONTS (Japanese Font)
+-------------------------------
+
+You can get the Japanese font pak from:
+
+http://yvt.jp/files/programs/osppaks/font-mplus-ja.pak
+
+This was converted from the M+ BDF font like this:
+
+	./BdfToOSFont ../../Resources/Gfx/Fonts/MPlus10Font double < \
+	~/Downloads/mplus_bitmap_fonts-2.2.4/fonts_j/mplus_j10r.bdf
+
+Note that you need to install a input method editor, or use Google Translator 
+or whatever to type Japanese characters.
+

Sources/Client/ChatWindow.cpp

@@ -26,6 +26,7 @@
 #include "World.h"
 #include <Core/Exception.h>
 #include <ctype.h>
+#include <Core/Math.h>
 
 
 namespace spades {
@@ -269,15 +270,20 @@ namespace spades {
 				float fade = ent.fade;
 				if(ent.timeFade < 1.f) { fade *= ent.timeFade; }
 				shadowColor.w = .5f * fade;
-				color.w *= fade;
-				std::string ch = "a";	//let's not make a new object for each character.
+				std::string ch = "aaaaaa";	//let's not make a new object for each character.
+				// note: UTF-8's longest character is 6 bytes
 				for(size_t i = 0; i < msg.size(); i++){
 					if(msg[i] == 13 || msg[i] == 10){
 						tx = 0.f; ty += lHeight;
-					}else if(msg[i] <= MsgColorMax){
+					}else if(msg[i] <= MsgColorMax && msg[i] >= 1){
 						if( msg[i] == MsgImage ) {
 							IImage* kill = NULL;
 							if( i+1 < msg.size() && (kill = imageForIndex(msg[i+1]))  ) {
+								Vector4 colorpm = color;
+								colorpm.x *= colorpm.w;
+								colorpm.y *= colorpm.w;
+								colorpm.z *= colorpm.w;
+								renderer->SetColorAlphaPremultiplied(colorpm);
 								renderer->DrawImage( kill, MakeVector2( tx + winX, ty + winY ) );
 								tx += kill->GetWidth();
 								++i;
@@ -286,10 +292,14 @@ namespace spades {
 							}
 						} else {
 							color = GetColor(msg[i]);
-							color.w *= fade;
 						}
 					}else{
-						ch[0] = msg[i];
+						size_t ln = 0;
+						GetCodePointFromUTF8String(msg, i, &ln);
+						ch.resize(ln);
+						for(size_t k = 0; k < ln; k++)
+							ch[k] = msg[i + k];
+						i += ln - 1;
 						font->DrawShadow(ch, MakeVector2(tx + winX, ty + winY), 1.f, color, shadowColor);
 						tx += font->Measure(ch).x;
 					}

Sources/Client/Client.cpp

@@ -143,19 +143,22 @@ namespace spades {
 										(const int *)UnsteadyOversteerMap,
 										30,
 										18);
+			static_cast<client::Quake3Font*>(designFont)->SetGlyphYRange(9.f, 24.f);
 			SPLog("Font 'Unsteady Oversteer' Loaded");
 			/*
 			textFont = new Quake3Font(renderer,
 										renderer->RegisterImage("Gfx/Fonts/UbuntuCondensed.tga"),
 										(const int*)UbuntuCondensedMap,
 										24,
-									  4);
+			 4);
+			 static_cast<client::Quake3Font*>(&*textFont)->SetGlyphYRange(4.f, 16.f);
 			SPLog("Font 'Ubuntu Condensed' Loaded");*/
 			textFont = new client::Quake3Font(renderer,
 										 renderer->RegisterImage("Gfx/Fonts/SquareFontModified.png"),
 										 (const int*)SquareFontMap,
 										 24,
-										 4);
+											  4);
+			static_cast<client::Quake3Font*>(textFont)->SetGlyphYRange(4.f, 16.f);
 			SPLog("Font 'SquareFont' Loaded");
 			
 			bigTextFont = new Quake3Font(renderer,
@@ -163,6 +166,7 @@ namespace spades {
 									  (const int*)UbuntuCondensedBigMap,
 									  48,
 										 8);
+			static_cast<client::Quake3Font*>(bigTextFont)->SetGlyphYRange(10.f, 33.f);
 			SPLog("Font 'Ubuntu Condensed (Large)' Loaded");
 			
 			world = NULL;

Sources/Client/IFont.cpp

@@ -19,20 +19,272 @@
  */
 
 #include "IFont.h"
+#include "IRenderer.h"
+#include <unordered_map>
+#include <Core/FileManager.h>
+#include <Core/IStream.h>
+#include <memory>
+#include <Core/Exception.h>
 
 namespace spades {
 	namespace client{
+		
+		class IImage;
+		
+		class FallbackFontManager;
+		struct FallbackFont;
+		
+		// FallbackFontManager cannot save IImage because multiple images
+		// come from different renderers.
+		class FallbackFontRenderer: public RefCountedObject {
+			std::vector<IImage *> images;
+			IRenderer *renderer;
+			FallbackFontManager *manager;
+			Handle<IImage> whiteImage; // used for last resort
+			
+			struct FindResult {
+				IImage *img;
+				float sizeInverse;
+				float advance;
+				int x, y, w, h;
+				int offX, offY;
+				FindResult():
+				img(nullptr), sizeInverse(0.f), advance(0.f),
+				x(0),y(0),w(0),h(0),offX(0),offY(0){}
+			};
+			FindResult FindGlyph(uint32_t);
+		protected:
+			virtual ~FallbackFontRenderer();
+		public:
+			
+			FallbackFontRenderer(IRenderer *renderer,
+								 FallbackFontManager *manager);
+			
+			void Draw(uint32_t unicode,
+					  Vector2 offset, float size,
+					  Vector4 color);
+			float Measure(uint32_t unicode, float size);
+			
+		};
+		
+		struct FallbackFont {
+		private:
+			
+		public:
+			struct GlyphInfo {
+				uint32_t unicode;
+				uint16_t x, y;
+				uint8_t w, h;
+				uint16_t advance;
+				uint16_t offX, offY;
+			};
+			std::string imagePath;
+			std::unordered_map<uint32_t, GlyphInfo> glyphs;
+			float fontSize;
+			float fontSizeInverse;
+			
+			FallbackFont(const std::string& path) {
+				std::unique_ptr<IStream> s(FileManager::OpenForReading(path.c_str()));
+				char buf[17];
+				buf[16] = 0;
+				if(s->Read(buf, 16) < 16) {
+					SPRaise("Reading %s: file truncated", path.c_str());
+				}
+				if(strcmp(buf, "OpenSpadesFontFl")) {
+					SPRaise("Reading %s: font file corrupted", path.c_str());
+				}
+				
+				uint32_t numGlyphs;
+				if(s->Read(&numGlyphs, 4) < 4) {
+					SPRaise("Reading %s: file truncated", path.c_str());
+				}
+				
+				uint32_t fontSize;
+				if(s->Read(&fontSize, 4) < 4) {
+					SPRaise("Reading %s: file truncated", path.c_str());
+				}
+				fontSize = (float)fontSize;
+				fontSizeInverse = 1.f / fontSize;
+				
+				std::vector<GlyphInfo> infos;
+				infos.resize(static_cast<std::size_t>(numGlyphs));
+				size_t siz = infos.size() * sizeof(GlyphInfo);
+				if(s->Read(infos.data(), siz) < siz) {
+					SPRaise("Reading %s: file truncated: trying to read %d byte(s)", path.c_str(), (int)siz);
+				}
+				
+				for(const auto& i: infos) {
+					glyphs[i.unicode] = i;
+				}
+				
+				// remove .ospfont and add .tga
+				std::string p = path.substr(0, path.size() - 8);
+				imagePath = p + ".tga";
+				if(FileManager::FileExists(imagePath.c_str())) {
+					return;
+				}
+				imagePath = p + ".png";
+				if(FileManager::FileExists(imagePath.c_str())) {
+					return;
+				}
+				SPRaise("Reading %s: failed to find %s.(png|tga)", path.c_str(), p.c_str());
+			}
+			
+		};
+		
+		class FallbackFontManager {
+			
+			FallbackFontManager() {
+				auto files = FileManager::EnumFiles("Gfx/Fonts");
+				for(const auto& file: files) {
+					if(file.rfind(".ospfont") != file.size() - 8) {
+						continue;
+					}
+					
+					std::string path = "Gfx/Fonts/" + file;
+					auto fnt = new FallbackFont(path);
+					fonts.push_back(fnt);
+				}
+			}
+		public:
+			std::vector<FallbackFont *> fonts;
+			
+			static FallbackFontManager *GetInstance() {
+				static FallbackFontManager *inst = nullptr;
+				if(inst == nullptr) {
+					inst = new FallbackFontManager();
+				}
+				return inst;
+			}
+			FallbackFontRenderer *CreateRenderer(IRenderer*r) {
+				return new FallbackFontRenderer(r,this);
+			}
+		};
+		
+		FallbackFontRenderer::FallbackFontRenderer(IRenderer *renderer,
+												   FallbackFontManager *manager):
+		renderer(renderer), manager(manager){
+			for(auto* font: manager->fonts) {
+				auto imgPath = font->imagePath;
+				auto *img = renderer->RegisterImage(imgPath.c_str()); // addref'd
+				images.push_back(img);
+			}
+			whiteImage.Set(renderer->RegisterImage("Gfx/White.tga"), false);
+		}
+		
+		FallbackFontRenderer::~FallbackFontRenderer() {
+			for(auto* img: images) {
+				img->Release();
+			}
+			images.clear();
+		}
+		
+		FallbackFontRenderer::FindResult FallbackFontRenderer::FindGlyph(uint32_t unicode) {
+			FindResult result;
+			for(size_t i = 0; i < manager->fonts.size(); i++) {
+				auto *font = manager->fonts[i];
+				const auto& it = font->glyphs.find(unicode);
+				if(it == font->glyphs.end()) {
+					continue;
+				}
+				result.img = images[i];
+				result.sizeInverse = font->fontSizeInverse;
+				result.advance = it->second.advance;
+				result.x = it->second.x;
+				result.y = it->second.y;
+				result.w = it->second.w;
+				result.h = it->second.h;
+				result.offX = it->second.offX;
+				result.offY = it->second.offY;
+				break;
+			}
+			return result;
+		}
+		
+		void FallbackFontRenderer::Draw(uint32_t unicode, spades::Vector2 offset,
+										float size, spades::Vector4 color) {
+			renderer->SetColorAlphaPremultiplied(color);
+			float x = offset.x;
+			float y = offset.y;
+			
+			auto glyph = FindGlyph(unicode);
+			if(glyph.img == nullptr) {
+				// no glyph found! draw box in the last resort
+				IImage *img = whiteImage;
+				renderer->DrawImage(img, AABB2(x, y, size, 1.f));
+				renderer->DrawImage(img, AABB2(x, y + size - 1.f, size, 1.f));
+				renderer->DrawImage(img, AABB2(x, y + 1.f, 1.f, size - 2.f));
+				renderer->DrawImage(img, AABB2(x + size - 1.f, y + 1.f, 1.f, size - 2.f));
+				return;
+			}
+			
+			if(glyph.w == 0) {
+				// null glyph.
+				return;
+			}
+			
+			float scale = size * glyph.sizeInverse;
+			
+			AABB2 inRect(glyph.x, glyph.y, glyph.w, glyph.h);
+			AABB2 outRect(glyph.offX, glyph.offY, glyph.w, glyph.h);
+			
+			// margin to make the border completely transparent
+			// (considering the OpenGL's linear interpolation)
+			inRect.min.x -= 0.5f;
+			inRect.min.y -= 0.5f;
+			outRect.min.x -= 0.5f;
+			outRect.min.y -= 0.5f;
+			inRect.max.x += 0.5f;
+			inRect.max.y += 0.5f;
+			outRect.max.x += 0.5f;
+			outRect.max.y += 0.5f;
+			
+			outRect.min *= scale;
+			outRect.max *= scale;
+			
+			outRect.min += offset;
+			outRect.max += offset;
+			
+			renderer->DrawImage(glyph.img, outRect, inRect);
+		}
+		
+		float FallbackFontRenderer::Measure(uint32_t unicode, float size) {
+			auto glyph = FindGlyph(unicode);
+			if(glyph.img == nullptr) {
+				// no glyph found! draw box in the last resort
+				return size + 1.f;
+			}
+			
+			float scale = size * glyph.sizeInverse;
+			return glyph.advance * scale;
+		}
+		
+		IFont::IFont(IRenderer *r) {
+			fallback.Set(FallbackFontManager::GetInstance()->CreateRenderer(r), false);
+		}
+		
 		IFont::~IFont()
 		{
 			//---
 		}
 		
+		float IFont::MeasureFallback(uint32_t unicodeCodePoint, float size) {
+			return fallback->Measure(unicodeCodePoint, size);
+		}
+		
+		void IFont::DrawFallback(uint32_t unicodeCodePoint,
+								 spades::Vector2 offset,
+								 float size,
+								 spades::Vector4 color) {
+			fallback->Draw(unicodeCodePoint, offset, size, color);
+		}
+		
 		void IFont::DrawShadow( const std::string& message, const Vector2& offset, float scale, const Vector4& color, const Vector4& shadowColor )
 		{
 			Draw( message, offset + MakeVector2(1,1), scale, shadowColor );
 			Draw( message, offset, scale, color );
 		}
-		
+	
 	}
 }
 

Sources/Client/IFont.h

@@ -25,12 +25,25 @@
 
 namespace spades {
 	namespace client{
+		class FallbackFontRenderer;
+		class IRenderer;
 		class IFont: public RefCountedObject {
+			Handle<FallbackFontRenderer> fallback;
 		protected:
 			virtual ~IFont();
+			
+			float MeasureFallback(uint32_t unicodeCodePoint, float size);
+			
+			/** Draws a unicode character using fallback fonts.
+			 * @param color Premultiplied alpha color value. */
+			void DrawFallback(uint32_t unicodeCodePoint, Vector2 offset, float size, Vector4 color);
+			
 		public:
-
+			IFont(IRenderer *);
 			virtual Vector2 Measure(const std::string&) = 0;
+			
+			/** Draws text.
+			 * @param color Non-premultiplied alpha color value. */
 			virtual void Draw(const std::string&, Vector2 offset, float scale, Vector4 color) = 0;
 			void DrawShadow( const std::string& message, const Vector2& offset, float scale, const Vector4& color, const Vector4& shadowColor );
 		};

Sources/Client/Quake3Font.cpp

@@ -25,7 +25,7 @@
 namespace spades {
 	namespace client {
 		Quake3Font::Quake3Font(IRenderer *r, IImage *tex, const int *mp, int gh, float sw)
-			:renderer(r), tex(tex),
+			:IFont(r), renderer(r), tex(tex),
 			glyphHeight(gh), spaceWidth(sw)
 		{
 			SPADES_MARK_FUNCTION();
@@ -53,6 +53,9 @@ namespace spades {
 				
 				glyphs.push_back(info);
 			}
+			
+			yMin = 0.f;
+			yMax = (float)gh;
 		}
 		
 		Quake3Font::~Quake3Font(){
@@ -60,12 +63,24 @@ namespace spades {
 			tex->Release();
 		}
 		
+		void Quake3Font::SetGlyphYRange(float yMin, float yMax) {
+			this->yMin = yMin;
+			this->yMax = yMax;
+		}
+		
 		Vector2 Quake3Font::Measure(const std::string &txt) {
 			SPADES_MARK_FUNCTION();
 			
 			float x = 0.f, w = 0.f, h = (float)glyphHeight;
-			for(size_t i = 0; i < txt.size(); i++){
-				int ch = ((int)txt[i]) & 127;
+			for(size_t i = 0; i < txt.size();){
+				size_t chrLen = 0;
+				uint32_t ch = GetCodePointFromUTF8String(txt, i, &chrLen);
+				SPAssert(chrLen > 0);
+				i += chrLen;
+				if(ch >= 128) {
+					goto fallback;
+				}
+				
 				
 				if(ch == 13 || ch == 10){
 					// new line
@@ -74,17 +89,26 @@ namespace spades {
 					continue;
 				}
 				
-				SPAssert(ch >= 0); SPAssert(ch < 128);
-				const GlyphInfo& info = glyphs[ch];
-				
-				if(info.type == Invalid)
-					continue;
-				else if(info.type == Space){
-					x += spaceWidth;
-				}else if(info.type == Image ) {
-					x += info.imageRect.GetWidth();
+				SPAssert(ch < 128);
+				{
+					const GlyphInfo& info = glyphs[ch];
+					
+					if(info.type == Invalid)
+						goto fallback;
+					else if(info.type == Space){
+						x += spaceWidth;
+					}else if(info.type == Image ) {
+						x += info.imageRect.GetWidth();
+					}
+					
+					if(x > w){
+						w = x;
+					}
 				}
-				
+					
+				continue;
+			fallback:
+				x += MeasureFallback(ch, yMax - yMin);
 				if(x > w){
 					w = x;
 				}
@@ -105,8 +129,14 @@ namespace spades {
 			color *= a;
 			renderer->SetColorAlphaPremultiplied(color);
 			
-			for(size_t i = 0; i < txt.size(); i++){
-				int ch = ((int)txt[i]) & 127;
+			for(size_t i = 0; i < txt.size();){
+				size_t chrLen = 0;
+				uint32_t ch = GetCodePointFromUTF8String(txt, i, &chrLen);
+				SPAssert(chrLen > 0);
+				i += chrLen;
+				if(ch >= 128) {
+					goto fallback;
+				}
 				
 				if(ch == 13 || ch == 10){
 					// new line
@@ -115,19 +145,25 @@ namespace spades {
 					continue;
 				}
 				
-				SPAssert(ch >= 0); SPAssert(ch < 128);
-				const GlyphInfo& info = glyphs[ch];
-				
-				if(info.type == Invalid)
-					continue;
-				else if(info.type == Space){
-					x += spaceWidth;
-				}else if(info.type == Image ) {
-					AABB2 rt(x * scale + offset.x, y * scale + offset.y, info.imageRect.GetWidth() * scale, info.imageRect.GetHeight() * scale);
-					renderer->DrawImage(tex, rt, info.imageRect);
-					x += info.imageRect.GetWidth();
+				{
+					SPAssert(ch >= 0); SPAssert(ch < 128);
+					const GlyphInfo& info = glyphs[ch];
+					
+					if(info.type == Invalid)
+						goto fallback;
+					else if(info.type == Space){
+						x += spaceWidth;
+					}else if(info.type == Image ) {
+						AABB2 rt(x * scale + offset.x, y * scale + offset.y, info.imageRect.GetWidth() * scale, info.imageRect.GetHeight() * scale);
+						renderer->DrawImage(tex, rt, info.imageRect);
+						x += info.imageRect.GetWidth();
+					}
 				}
-				
+				continue;
+			fallback:
+				DrawFallback(ch, MakeVector2(x, y + yMin) * scale + offset,
+							 (yMax - yMin) * scale, color);
+				x += MeasureFallback(ch, (yMax - yMin) * scale);
 			}
 		}
 	}

Sources/Client/Quake3Font.h

@@ -45,6 +45,8 @@ namespace spades {
 			int glyphHeight;
 			std::vector<GlyphInfo> glyphs;
 			float spaceWidth;
+			
+			float yMin, yMax;
 		protected:
 			virtual ~Quake3Font();
 		public:
@@ -59,6 +61,7 @@ namespace spades {
 							  Vector2 offset,
 							  float scale,
 							  Vector4 color);
+			void SetGlyphYRange(float yMin, float yMax);
 		};
 	}
 }

Sources/Core/Math.cpp

@@ -510,6 +510,64 @@ namespace spades {
 		return true;
 	}
 	
+	uint32_t GetCodePointFromUTF8String(const std::string& str, size_t start, size_t *outNumBytes) {
+		if(start >= str.size()) {
+			if(outNumBytes) *outNumBytes = 0;
+			return 0;
+		}
+		if((str[start] & 0x80) == 0) {
+			if(outNumBytes) *outNumBytes = 1;
+			return (uint32_t)str[start];
+		}
+		
+		// WARNING: start <= str.size() - 2 is bad (this leads to security holes)
+		uint32_t ret;
+		if((str[start] & 0xe0) == 0xc0 && start + 2 <= str.size()) {
+			if(outNumBytes) *outNumBytes = 2;
+			ret = (str[start] & 0x1f) << 6;
+			ret |= (str[start + 1] & 0x3f);
+			return ret;
+		}
+		if((str[start] & 0xf0) == 0xe0 && start + 3 <= str.size()) {
+			if(outNumBytes) *outNumBytes = 3;
+			ret = (str[start] & 0xf) << 12;
+			ret |= (str[start + 1] & 0x3f) << 6;
+			ret |= (str[start + 2] & 0x3f);
+			return ret;
+		}
+		if((str[start] & 0xf8) == 0xf0 && start + 4 <= str.size()) {
+			if(outNumBytes) *outNumBytes = 4;
+			ret = (str[start] & 0x7) << 18;
+			ret |= (str[start + 1] & 0x3f) << 12;
+			ret |= (str[start + 2] & 0x3f) << 6;
+			ret |= (str[start + 3] & 0x3f);
+			return ret;
+		}
+		if((str[start] & 0xfc) == 0xf8 && start + 5 <= str.size()) {
+			if(outNumBytes) *outNumBytes = 5;
+			ret = (str[start] & 0x3) << 24;
+			ret |= (str[start + 1] & 0x3f) << 18;
+			ret |= (str[start + 2] & 0x3f) << 12;
+			ret |= (str[start + 3] & 0x3f) << 6;
+			ret |= (str[start + 4] & 0x3f);
+			return ret;
+		}
+		if((str[start] & 0xfe) == 0xfc && start + 6 <= str.size()) {
+			if(outNumBytes) *outNumBytes = 6;
+			ret = (str[start] & 0x1) << 30;
+			ret |= (str[start + 1] & 0x3f) << 24;
+			ret |= (str[start + 2] & 0x3f) << 18;
+			ret |= (str[start + 3] & 0x3f) << 12;
+			ret |= (str[start + 4] & 0x3f) << 6;
+			ret |= (str[start + 5] & 0x3f);
+			return ret;
+		}
+		
+		// invalid UTF8
+		if(outNumBytes) *outNumBytes = 1;
+		return (uint32_t)str[start];
+	}
+	
 	float GetRandom() {
 		const double factor = 1.f / ((double)RAND_MAX + 1.);
 		return (float)((double)rand() * factor);

Sources/Core/Math.h

@@ -842,6 +842,8 @@ namespace spades {
 	std::vector<std::string> Split(const std::string&, const std::string& sep);
 
 	std::vector<std::string> SplitIntoLines(const std::string&);
+	
+	uint32_t GetCodePointFromUTF8String(const std::string&, size_t start = 0, size_t *outNumBytes = nullptr);
 
 	std::string TrimSpaces(const std::string&);
 	

Sources/Gui/MainScreen.cpp

@@ -49,6 +49,7 @@ namespace spades {
 									  (const int*)SquareFontMap,
 									  24,
 									  4);
+			static_cast<client::Quake3Font*>(&*font)->SetGlyphYRange(4.f, 16.f);
 			SPLog("Font 'SquareFont' Loaded");
 			
 			helper = new MainScreenHelper(this);

Sources/Tools/a.out.dSYM/Contents/Info.plist

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+	<dict>
+		<key>CFBundleDevelopmentRegion</key>
+		<string>English</string>
+		<key>CFBundleIdentifier</key>
+		<string>com.apple.xcode.dsym.a.out</string>
+		<key>CFBundleInfoDictionaryVersion</key>
+		<string>6.0</string>
+		<key>CFBundlePackageType</key>
+		<string>dSYM</string>
+		<key>CFBundleSignature</key>
+		<string>????</string>
+		<key>CFBundleShortVersionString</key>
+		<string>1.0</string>
+		<key>CFBundleVersion</key>
+		<string>1</string>
+	</dict>
+</plist>