public class XmlOrJsonConverterFactory extends Converter.Factory {
    final Converter.Factory xml = SimpleXmlConverterFactory.create();
    final Converter.Factory gson = GsonConverterFactory.create();

    @Override
    public Converter<ResponseBody, ?> responseBodyConverter(
            Type type, Annotation[] annotations, Retrofit retrofit) {

        // Retrofit gives us all the annotations so we just need to check
        for (Annotation annotation : annotations) {
            if (annotation.annotationType() == Xml.class) {
                return xml.responseBodyConverter(type, annotations, retrofit);
            }
            if (annotation.annotationType() == Json.class) {
                return gson.responseBodyConverter(type, annotations, retrofit);
            }
        }
        // There is no annotation so we cannot handle it
        return null;
    }
}

// and my service would be something like
@Json
@POST("/some/rest/api")
Call<Stuff> getStuff();

@Xml
@POST("/some/soap/api")
Call<OtherStuff getOtherStuff();

// and we would simple build retrofit with
new Retrofit.Builder()
            .addConverterFactory(new XmlOrJsonConverterFactory())
            .build();